package tv.yunxi.imgfmt; import android.content.Context; import android.content.res.AssetManager; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.opengl.GLES20; import android.opengl.GLSurfaceView; import android.opengl.Matrix; import android.util.Log; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.nio.IntBuffer; import java.util.Arrays; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; /** * @author zhaozl * @date 2025/1/20 17:45 */ public class ImageRenderer implements GLSurfaceView.Renderer { private Context mContext; public ImageRenderer(Context context) { mContext = context; mVertices = ByteBuffer.allocateDirect(mVerticesData.length * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer(); mVertices.put(mVerticesData).position(0); Matrix.setIdentityM(mSTMatrix, 0); Log.v(TAG, "mSTMatrix: " + Arrays.toString(mSTMatrix)); Matrix.setIdentityM(mMMatrix, 0); // Matrix.rotateM(mMMatrix, 0, 20, 0, 1, 0); } @Override public void onSurfaceChanged(GL10 glUnused, int width, int height) { Log.v(TAG, "onSurfaceChanged [" + width + ", " + height + "]"); // Ignore the passed-in GL10 interface, and use the GLES20 // class's static methods instead. GLES20.glViewport(0, 0, width, height); mRatio = (float) width / height; // Log.v(TAG, "mProjMatrix 1: " + Arrays.toString(mProjMatrix)); // Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7); Matrix.frustumM(mProjMatrix, 0, -1, 1, -1, 1, 3, 7); Log.v(TAG, "mProjMatrix: " + Arrays.toString(mProjMatrix)); } @Override public void onSurfaceCreated(GL10 glUnused, EGLConfig config) { Log.v(TAG, "onSurfaceCreated"); // Ignore the passed-in GL10 interface, and use the GLES20 // class's static methods instead. /* Set up alpha blending and an Android background color */ GLES20.glEnable(GLES20.GL_BLEND); GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f); /* Set up shaders and handles to their variables */ String fragmentShader = loadShaderFromAssets("FRAGMENT_SHADER_RGB2NV21.glsl"); mProgram = createProgram(mVertexShader, fragmentShader); // mProgram = createProgram(mVertexShader, mFragmentShader); if (mProgram == 0) { return; } // 纹理坐标相关参数 // 获取uWidth location muWidthHandle = GLES20.glGetUniformLocation(mProgram, "uWidth"); checkGlError("glGetUniformLocation uWidth"); if (muWidthHandle == -1) { throw new RuntimeException("Could not get attrib location for uWidth"); } // 获取uHeight location muHeightHandle = GLES20.glGetUniformLocation(mProgram, "uHeight"); checkGlError("glGetUniformLocation uHeight"); if (muHeightHandle == -1) { throw new RuntimeException("Could not get attrib location for uHeight"); } // 顶点坐标相关参数 maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); checkGlError("glGetAttribLocation aPosition"); if (maPositionHandle == -1) { throw new RuntimeException("Could not get attrib location for aPosition"); } maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); checkGlError("glGetAttribLocation aTextureCoord"); if (maTextureHandle == -1) { throw new RuntimeException("Could not get attrib location for aTextureCoord"); } muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); checkGlError("glGetUniformLocation uMVPMatrix"); if (muMVPMatrixHandle == -1) { throw new RuntimeException("Could not get attrib location for uMVPMatrix"); } muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); checkGlError("glGetUniformLocation uSTMatrix"); if (muMVPMatrixHandle == -1) { throw new RuntimeException("Could not get attrib location for uSTMatrix"); } checkGlError("glGetUniformLocation uCRatio"); if (muMVPMatrixHandle == -1) { throw new RuntimeException("Could not get attrib location for uCRatio"); } createTexture(); createFrameBuffer(); Log.v(TAG, "mVMatrix 1: " + Arrays.toString(mVMatrix)); Matrix.setLookAtM(mVMatrix, 0, 0, 0, 3f, 0f, 0f, 0f, 0f, 1.0f, 0.0f); Log.v(TAG, "mVMatrix: " + Arrays.toString(mVMatrix)); mPixelBuffer = ByteBuffer.allocateDirect(NV21_SIZE); // RGBA format mPixelBuffer.order(ByteOrder.nativeOrder()); } @Override public void onDrawFrame(GL10 glUnused) { // 绑定帧缓冲区 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffer); checkGlError("glBindFramebuffer mFrameBuffer: " + mFrameBuffer); renderTexture(); GLES20.glReadPixels(0, 0, mWidth, mHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPixelBuffer); checkGlError("glReadPixels"); mPixelBuffer.rewind(); mPixelBuffer.get(mImageBytes); saveImage(mImageBytes, "nv21"); GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); checkGlError("glBindFramebuffer 0"); renderTexture(); } private void renderTexture() { GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); GLES20.glUseProgram(mProgram); checkGlError("glUseProgram"); GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureID); // 设置顶点坐标 mVertices.position(VERTICES_DATA_POS_OFFSET); GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false, VERTICES_DATA_STRIDE_BYTES, mVertices); checkGlError("glVertexAttribPointer maPosition"); GLES20.glEnableVertexAttribArray(maPositionHandle); checkGlError("glEnableVertexAttribArray maPositionHandle"); // 设置纹理坐标 mVertices.position(VERTICES_DATA_UV_OFFSET); GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false, VERTICES_DATA_STRIDE_BYTES, mVertices); checkGlError("glVertexAttribPointer maTextureHandle"); GLES20.glEnableVertexAttribArray(maTextureHandle); checkGlError("glEnableVertexAttribArray maTextureHandle"); // mMVPMatrix = mVMatrix * mMMatrix Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0); // mMVPMatrix = mProjMatrix * mMVPMatrix Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0); GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0); checkGlError("glUniformMatrix4fv mMVPMatrix"); GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0); checkGlError("glUniformMatrix4fv mSTMatrix"); // RGB2NV21 需要设置uWidth和uHeight if (muWidthHandle != -1) { // 设置uWidth GLES20.glUniform1f(muWidthHandle, mWidth); checkGlError("glUniform1f muWidthHandle"); } if (muHeightHandle != -1) { // 设置uHeight GLES20.glUniform1f(muHeightHandle, mHeight); checkGlError("glUniform1f muHeightHandle"); } GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); checkGlError("glDrawArrays"); } private void createTexture() { ByteBuffer imageBuffer = loadBytesFromAssets("image/fish.rgba"); // 创建纹理 int[] textures = new int[1]; GLES20.glGenTextures(1, textures, 0); mTextureID = textures[0]; GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureID); checkGlError("glBindTexture mTextureID"); // GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, bitmap, 0); GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, mWidth, mHeight, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, imageBuffer); checkGlError("texImage2D mTextureID"); // Can't do mipmapping with camera source GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); // Clamp to edge is the only option GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); checkGlError("glTexParameteri mTextureID: " + mTextureID); } private void createFrameBuffer() { // 创建帧缓冲区 int[] frameBuffers = new int[1]; GLES20.glGenFramebuffers(1, frameBuffers, 0); mFrameBuffer = frameBuffers[0]; // 绑定帧缓冲区 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBuffer); checkGlError("glBindFramebuffer mFrameBuffer: " + mFrameBuffer); // 创建纹理 int[] textures = new int[1]; GLES20.glGenTextures(1, textures, 0); mFrameTextureID = textures[0]; GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mFrameTextureID); checkGlError("glBindTexture mFrameTextureID"); GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, mWidth, mHeight, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null); checkGlError("glTexImage2D mFrameTextureID"); GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, mFrameTextureID, 0); checkGlError("glFramebufferTexture2D mFrameTextureID"); // 检查帧缓冲区是否完整 if (GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) != GLES20.GL_FRAMEBUFFER_COMPLETE) { Log.e(TAG, "Framebuffer is not complete"); } // 解绑帧缓冲区 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); checkGlError("glBindFramebuffer 0"); } private int loadShader(int shaderType, String source) { int shader = GLES20.glCreateShader(shaderType); if (shader != 0) { GLES20.glShaderSource(shader, source); GLES20.glCompileShader(shader); int[] compiled = new int[1]; GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); if (compiled[0] == 0) { Log.e(TAG, "Could not compile shader " + shaderType + ":"); Log.e(TAG, GLES20.glGetShaderInfoLog(shader)); GLES20.glDeleteShader(shader); shader = 0; } } return shader; } private int createProgram(String vertexSource, String fragmentSource) { int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); if (vertexShader == 0) { return 0; } int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); if (pixelShader == 0) { return 0; } int program = GLES20.glCreateProgram(); if (program != 0) { GLES20.glAttachShader(program, vertexShader); checkGlError("glAttachShader"); GLES20.glAttachShader(program, pixelShader); checkGlError("glAttachShader"); GLES20.glLinkProgram(program); int[] linkStatus = new int[1]; GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); if (linkStatus[0] != GLES20.GL_TRUE) { Log.e(TAG, "Could not link program: "); Log.e(TAG, GLES20.glGetProgramInfoLog(program)); GLES20.glDeleteProgram(program); program = 0; } } return program; } private void checkGlError(String op) { int error; while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { Log.e(TAG, op + ": glError " + error); throw new RuntimeException(op + ": glError " + error); } } private static final int FLOAT_SIZE_BYTES = 4; private static final int VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; private static final int VERTICES_DATA_POS_OFFSET = 0; private static final int VERTICES_DATA_UV_OFFSET = 3; // private final float[] mVerticesData = { // // X, Y, Z, U, V // -1.0f, -1.0f, 0, 0.f, 0.f, // 左下 // 1.0f, -1.0f, 0, 1.f, 0.f, // 右下 // -1.0f, 1.0f, 0, 0.f, 1.f, // 左上 // 1.0f, 1.0f, 0, 1.f, 1.f, // 右上 // }; private final float[] mVerticesData = { // X, Y, Z, U, V -1.0f, -1.0f, 0 /* 左下 */, 0.f, 1.f, /* 左上 */ 1.0f, -1.0f, 0 /* 右下 */, 1.f, 1.f, /* 右上 */ -1.0f, 1.0f, 0 /* 左上 */, 0.f, 0.f, /* 左下 */ 1.0f, 1.0f, 0 /* 右上 */, 1.f, 0.f, /* 右下 */ }; private FloatBuffer mVertices; private final String mVertexShader = "uniform mat4 uMVPMatrix;\n" + "uniform mat4 uSTMatrix;\n" + "attribute vec4 aPosition;\n" + "attribute vec4 aTextureCoord;\n" + "varying vec2 vTextureCoord;\n" + "void main() {\n" + " gl_Position = uMVPMatrix * aPosition;\n" + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + "}\n"; private final String mFragmentShader = "precision mediump float;\n" + "varying vec2 vTextureCoord;\n" + "uniform sampler2D sTexture;\n" + "void main() {\n" + " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + "}\n"; /* * Model-View-Projection Matrix * 这是一个组合矩阵，用于将模型坐标转换为屏幕坐标。它是模型矩阵、视图矩阵和投影矩阵的乘积。 * 作用：将对象从其本地坐标系转换到屏幕坐标系，经过模型变换、视图变换和投影变换。 */ private float[] mMVPMatrix = new float[16]; /* * Projection Matrix * 用于定义视锥体的投影变换。它将3D坐标投影到2D平面上。 * 作用：控制视野的宽度、高度和深度，决定了场景中物体的透视效果。 */ private float[] mProjMatrix = new float[16]; /* * Model Matrix * 用于将对象从其本地坐标系转换到世界坐标系。它包含了对象的平移、旋转和缩放信息。 * 作用：定义对象在世界坐标系中的位置和姿态。 */ private float[] mMMatrix = new float[16]; /* * View Matrix * 用于将世界坐标系转换到摄像机坐标系。它定义了摄像机的位置和方向。 * 作用：模拟摄像机的视角，决定了从哪个角度观察场景。 */ private float[] mVMatrix = new float[16]; /* * Surface Texture Matrix * 用于处理纹理坐标的变换，特别是在使用SurfaceTexture时。 * 作用：调整纹理坐标以适应纹理的变换，比如旋转、缩放等。 */ private float[] mSTMatrix = new float[16]; private int mProgram; private int mTextureID; private int mFrameBuffer; private int mFrameTextureID; private int muMVPMatrixHandle; private int muSTMatrixHandle; private int maPositionHandle; private int maTextureHandle; private int muWidthHandle = -1; private int muHeightHandle = -1; private ByteBuffer mPixelBuffer; private float mRatio = 1.0f; private static final int mWidth = 1920; private static final int mHeight = 1080; private static final int RGBA_SIZE = mWidth * mHeight * 4; private static final int NV21_SIZE = RGBA_SIZE * 3 / 8; private byte[] mImageBytes = new byte[NV21_SIZE]; private static final String TAG = "ImageRenderer"; private Bitmap loadBitmapFromAssets(String fileName) { AssetManager assetManager = mContext.getAssets(); try (InputStream inputStream = assetManager.open(fileName)) { return BitmapFactory.decodeStream(inputStream); } catch (IOException e) { Log.e(TAG, "Error loading bitmap from assets: " + e.getMessage()); return null; } } private ByteBuffer loadBytesFromAssets(String fileName) { AssetManager assetManager = mContext.getAssets(); try (InputStream inputStream = assetManager.open(fileName)) { return ByteBuffer.wrap(inputStream.readAllBytes()); } catch (IOException e) { Log.e(TAG, "Error loading bytes from assets: " + e.getMessage()); return null; } } private String loadShaderFromAssets(String fileName) { AssetManager assetManager = mContext.getAssets(); try (InputStream inputStream = assetManager.open("glsl/" + fileName)) { byte[] buffer = new byte[inputStream.available()]; inputStream.read(buffer); return new String(buffer); } catch (IOException e) { Log.e(TAG, "Error loading shader from assets: " + e.getMessage()); return null; } } private int index = 0; private void saveImage(byte[] buffer, String suffix) { Log.v(TAG, "saveImage buffer length: " + buffer.length); FileOutputStream fos = null; try { // String path = "/storage/9016-4EF8/dump_rgba"; String path = "/data/data/tv.yunxi.imgfmt/files/dump_rgba"; File filePath = new File(path); if (!filePath.exists()) { filePath.mkdirs(); } File file = new File(path + "/image_" + index++ + "." + suffix); if (!file.exists()) { file.createNewFile(); } // 写入文件 fos = new FileOutputStream(file); fos.write(buffer); fos.flush(); fos.close(); } catch (Exception e) { e.printStackTrace(); } finally { if (fos != null) { try { fos.close(); } catch (IOException e) { e.printStackTrace(); } } } } }