From realtime CameraX filters to General Purpose GPU computing in simple steps, Droidcon London 2024

GpGpuRenderer.kt

package io.mightyseal.shadercamera.gpgpu

import android.graphics.Bitmap
import android.opengl.EGL14
import android.opengl.EGLConfig
import android.opengl.EGLExt
import android.opengl.GLES31
import android.opengl.GLES32
import android.opengl.GLUtils
import io.mightyseal.shadercamera.camera.checkGlErrorOrThrow
import io.mightyseal.shadercamera.camera.createByteBuffer
import io.mightyseal.shadercamera.camera.createRawBufferOfInts
import timber.log.Timber
import java.nio.ByteBuffer
import java.nio.ByteOrder

class GpGpuComputeImage(
    private val bitmap: Bitmap,
    private val shaderProvider: () -> String
) {

    init {
        val eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
        if (eglDisplay == EGL14.EGL_NO_DISPLAY) {
            throw IllegalStateException("Unable to get EGL14 display ${GLUtils.getEGLErrorString(EGL14.eglGetError())}")
        }

        val version = IntArray(2)
        if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) {
            throw IllegalStateException("Unable to initialize EGL14 ${GLUtils.getEGLErrorString(EGL14.eglGetError())}")
        }

        val configs = arrayOfNulls<EGLConfig>(1)
        val numConfigs = IntArray(1)

        val attribToChooseConfig = intArrayOf(
            EGL14.EGL_RED_SIZE, 8,
            EGL14.EGL_GREEN_SIZE, 8,
            EGL14.EGL_BLUE_SIZE, 8,
            EGL14.EGL_ALPHA_SIZE, 8,
            EGL14.EGL_DEPTH_SIZE, 0,
            EGL14.EGL_STENCIL_SIZE, 0,
            EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
            EGLExt.EGL_RECORDABLE_ANDROID, EGL14.EGL_TRUE,
            EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT or EGL14.EGL_PBUFFER_BIT,
            EGL14.EGL_NONE
        )

        if (!EGL14.eglChooseConfig(
                eglDisplay,
                attribToChooseConfig,
                0,
                configs,
                0,
                configs.size,
                numConfigs,
                0
            )
        ) {
            throw IllegalStateException("Unable to find a suitable EGLConfig")
        }

        val config = configs[0] ?: throw IllegalStateException("EGLConfig was not initialized")
        val attribToCreateContext = intArrayOf(
            EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE
        )
        val eglContext = EGL14.eglCreateContext(
            eglDisplay, config, EGL14.EGL_NO_CONTEXT,
            attribToCreateContext, 0
        )

        // Confirm with query
        val values = IntArray(1)
        EGL14.eglQueryContext(
            eglDisplay,
            eglContext,
            EGL14.EGL_CONTEXT_CLIENT_VERSION,
            values,
            0
        )

        EGL14.eglMakeCurrent(eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, eglContext)
    }

    fun compute(): OutputImageData {
        val shader = initShader()

        val computeProgram = GLES32.glCreateProgram()
        checkGlErrorOrThrow("glCreateProgram")
        GLES32.glAttachShader(computeProgram, shader)
        checkGlErrorOrThrow("glAttachShader")
        GLES32.glLinkProgram(computeProgram)
        val linkStatus = IntArray(1)
        GLES32.glGetProgramiv(computeProgram, GLES31.GL_LINK_STATUS, linkStatus, 0)

        if (linkStatus[0] != GLES32.GL_TRUE) {
            val error = GLES32.glGetProgramInfoLog(computeProgram)
            Timber.e("Program compilation failed $error")
            GLES32.glDeleteProgram(computeProgram)
            throw IllegalStateException("Program link failed: $error")
        }

        GLES32.glUseProgram(computeProgram)

        setUniformInputs(bitmap.width, bitmap.height)
        useInput(GpuComputeImageData(bitmap))
        val (ssboId, size) = useOutput(bitmap.height * bitmap.width * Int.SIZE_BYTES)

        GLES32.glDispatchCompute(bitmap.width, bitmap.height, 1)
        GLES32.glMemoryBarrier(GLES32.GL_SHADER_STORAGE_BARRIER_BIT)

        checkGlErrorOrThrow("glUniform1i width")

        checkGlErrorOrThrow("glDispatchCompute")
        val output = readResult(ssboId, size, bitmap.width, bitmap.height)
        checkGlErrorOrThrow("readResult")

        return output
    }

    private fun initShader(): Int {
        val shader = GLES32.glCreateShader(GLES32.GL_COMPUTE_SHADER)
        checkGlErrorOrThrow("glCreateShader")
        GLES32.glShaderSource(shader, shaderProvider())
        GLES32.glCompileShader(shader)
        checkGlErrorOrThrow("glCompileShader")

        val compiled = IntArray(1)
        GLES32.glGetShaderiv(shader, GLES32.GL_COMPILE_STATUS, compiled, 0)
        if (compiled[0] == GLES32.GL_FALSE) {
            val error = GLES32.glGetShaderInfoLog(shader)
            Timber.e(error)
            GLES32.glDeleteShader(shader)
            throw IllegalStateException("Shader compilation failed: $error")
        }

        return shader
    }

    private fun useOutput(imageSize: Int): Pair<Int, Int> {
        val (buffer, size) = OutputImageData.createBuffer(imageSize)

        val bo = intArrayOf(0)
        GLES32.glGenBuffers(1, bo, 0)
        GLES32.glBindBuffer(GLES32.GL_SHADER_STORAGE_BUFFER, bo[0])
        GLES32.glBufferData(
            GLES32.GL_SHADER_STORAGE_BUFFER,
            size,
            buffer,
            GLES32.GL_STATIC_READ
        )

        GLES32.glBindBufferBase(GLES32.GL_SHADER_STORAGE_BUFFER, OUTPUT_BINDING_INDEX, bo[0])

        checkGlErrorOrThrow("useOutput")

        return Pair(bo[0], size)
    }

    private fun setUniformInputs(width: Int, height: Int) {
        GLES32.glUniform1i(WIDTH_BINDING_INDEX, width)
        checkGlErrorOrThrow("glUniform1i width")
        GLES32.glUniform1i(HEIGHT_BINDING_INDEX, height)
        checkGlErrorOrThrow("glUniform1i height")
    }

    private fun useInput(data: GpuComputeImageData) {
        val (inputBuffer, bufferSize) = data.toBuffer()

        val bo = intArrayOf(0)
        GLES32.glGenBuffers(1, bo, 0)
        GLES32.glBindBuffer(GLES32.GL_SHADER_STORAGE_BUFFER, bo[0])
        GLES32.glBufferData(
            GLES32.GL_SHADER_STORAGE_BUFFER,
            bufferSize,
            inputBuffer,
            GLES32.GL_STATIC_READ
        )

        GLES32.glBindBufferBase(GLES32.GL_SHADER_STORAGE_BUFFER, INPUT_BINDING_INDEX, bo[0])

        checkGlErrorOrThrow("useInput")
    }

    private fun readResult(ssboId: Int, size: Int, width: Int, height: Int): OutputImageData {
        GLES32.glBindBuffer(GLES31.GL_SHADER_STORAGE_BUFFER, ssboId)
        val buffer = GLES32.glMapBufferRange(
            GLES31.GL_SHADER_STORAGE_BUFFER,
            0,
            size,
            GLES31.GL_MAP_READ_BIT
        ) as ByteBuffer

        buffer.order(ByteOrder.nativeOrder())
        val intBuffer = buffer.asIntBuffer()
        val outBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
        outBitmap.copyPixelsFromBuffer(intBuffer)

        val result = OutputImageData(outBitmap)

        GLES32.glUnmapBuffer(GLES31.GL_SHADER_STORAGE_BUFFER)
        GLES32.glBindBuffer(GLES31.GL_SHADER_STORAGE_BUFFER, 0)

        return result
    }

    companion object {
        const val INPUT_BINDING_INDEX = 0
        const val OUTPUT_BINDING_INDEX = 1
        const val WIDTH_BINDING_INDEX = 2
        const val HEIGHT_BINDING_INDEX = 3
    }
}

private data class GpuComputeImageData(
    val bitmap: Bitmap
)

private fun GpuComputeImageData.toBuffer(): Pair<ByteBuffer, Int> {
    val size = bitmap.height * bitmap.width * Int.SIZE_BYTES

    val buffer = createByteBuffer(size)
    bitmap.copyPixelsToBuffer(buffer)
    buffer.position(0)

    return buffer to size
}

data class OutputImageData(
    val bitmap: Bitmap
) {
    companion object
}

private fun OutputImageData.Companion.createBuffer(size: Int): Pair<ByteBuffer, Int> {
    val buffer = createRawBufferOfInts(size)
    return buffer to size
}

image_grayscale_shader.comp

#version 320 es

layout (local_size_x = 1, local_size_y = 1, local_size_z = 1) in;

layout(std430, binding=0) readonly buffer InputImage {
    int pixels[];
} inputImage;

layout(std430, binding=1) writeonly buffer OutputImage {
    int pixels[];
} outputImage;

layout(location = 2) uniform int width;
layout(location = 3) uniform int height;

int transform(int x, int y) {
    return inputImage.pixels[y * width + x];
}

vec3 grayscale(vec3 color) {
    return vec3(color.r * 0.2126 + color.g * 0.7152 + color.b * 0.0722);
}

void main() {
    ivec2 storePos = ivec2(gl_GlobalInvocationID.xy);
    if (storePos.x >= width || storePos.y >= height) return;
    int pix = transform(storePos.x, storePos.y);

    // ARGB
    int alpha = (pix >> 24) & 0xff;
    float red = float((pix >> 16) & 0xff);
    float green = float((pix >> 8) & 0xff);
    float blue = float(pix & 0xff);

    vec3 floatColor = vec3(red, green, blue);
    vec3 transformed = grayscale(floatColor);
    int outPix = alpha << 24 | int(transformed.r) << 16 | int(transformed.g) << 8 | int(transformed.b);

    outputImage.pixels[storePos.y * width + storePos.x] = outPix;
}