Development of 3D Scan Application by using DepthVision (ToF) Camera

Mobile Galaxy Camera
1

I 'm a researcher and I want to develop a 3D scan application of scanning foot by using the DepthVision (ToF) camera on Note10+. However, there are no official SDK for controlling the DepthVision camera. In other topics, there are some methods for getting the depth image of the DepthVision camera. Nevertheless, I also need the intrinsic parameters of the DepthVision camera and the color camera and the extrinsic parameters between them to reconstruct a 3D point cloud of the foot with texture. Does anyone know how to get the above-mentioned parameters? And I also want to know is there possible that official SDK for controlling the DepthVision camera will appear in the future?

2

Hi,
Would you please check below resource and let us know if these help you?

And,

Camera extrinsic and intrinsic parameters are exposed via Camera2 APIs. Any regular device on the market will advertise a valid value.

Please check below APIs,

Thanks

3

Sorry for late reply. Thank you very much for replying my question. I have used camera2 API for capturing the depth image. However, I find that in preview mode, I can capture depth image correctly. But, in capture mode, the captured depth image is always empty. Do you know why? And how to fix it?

4

The following is my code for depth camera:

public class AndroidDepthCamera
{
public TextureView depthPreviewTextureView;
public TextureView depthPreviewTextureView2;

private MainActivity mMainActivity;
private CameraManager camMgr;
private WindowManager windowMgr;
private String packageName;

private CameraDevice depthCameraDevice = null;
private int depthImgWidth = 640, depthImgHeight = 480;
private CaptureRequest.Builder depthPreviewBuilder = null;
private CameraCaptureSession depthCamPreviewCaptureSession = null;
private ImageReader depthPreviewImgReader;

private CameraCaptureSession depthCamTakePicCaptureSession = null;
private File coloredDepthImgFile;
private CaptureRequest.Builder depthCapBuilder;
private Handler depthTakePicBGHandler;
private ImageReader depthTakePicImgReader;
private Bitmap grayScaleDepthBitmapG;

public AndroidDepthCamera(MainActivity mMainActivity, CameraManager camMgr, WindowManager windowMgr, String packageName)
{
this.mMainActivity = mMainActivity;
this.camMgr = camMgr;
this.windowMgr = windowMgr;
this.packageName = packageName;

depthPreviewImgReader = ImageReader.newInstance(depthImgWidth, depthImgHeight, ImageFormat.DEPTH16, 2);
depthPreviewImgReader.setOnImageAvailableListener(depthImgReaderPreviewOnImageAvailable, null);

}

public void openCamera()
{
try
{
String depthCamId = camMgr.getCameraIdList()[4];
CameraCharacteristics depthCamChar = camMgr.getCameraCharacteristics(depthCamId);

  StreamConfigurationMap colorStrConfigMap = depthCamChar.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
  
  if(ContextCompat.checkSelfPermission(mMainActivity, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED)
    camMgr.openCamera(depthCamId, depthCameraStateCallback, null);
}
catch (CameraAccessException e)
{
  e.printStackTrace();
}

}

private CameraDevice.StateCallback depthCameraStateCallback = new CameraDevice.StateCallback()
{
@Override
public void onOpened(@NonNull CameraDevice cameraDevice)
{
depthCameraDevice = cameraDevice;
startDepthCamPreview();
}

@Override
public void onDisconnected(@NonNull CameraDevice cameraDevice)
{
  Toast.makeText(mMainActivity, "Cannot use color camera!", Toast.LENGTH_LONG).show();
}

@Override
public void onError(@NonNull CameraDevice cameraDevice, int i)
{
  Toast.makeText(mMainActivity, "Color camera opening error!", Toast.LENGTH_LONG).show();
}

};

private void startDepthCamPreview()
{
try
{
depthPreviewBuilder = depthCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
}
catch (CameraAccessException e)
{
e.printStackTrace();
}

depthPreviewBuilder.set(CaptureRequest.JPEG_ORIENTATION, 0);
Range<Integer> fpsRange = new Range<>(15, 30);
depthPreviewBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
depthPreviewBuilder.addTarget(depthPreviewImgReader.getSurface());

try
{
  depthCameraDevice.createCaptureSession(Arrays.asList(depthPreviewImgReader.getSurface()), depthCamPreviewCaptureSessionStateCallback, null);
}
catch (CameraAccessException e)
{
  e.printStackTrace();
}

}

private CameraCaptureSession.StateCallback depthCamPreviewCaptureSessionStateCallback = new CameraCaptureSession.StateCallback()
{
@Override
public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession)
{
closeAllCameraCaptureSession();

  depthCamPreviewCaptureSession = cameraCaptureSession;
  depthPreviewBuilder.set(CaptureRequest.CONTROL_MODE,  CameraMetadata.CONTROL_MODE_AUTO);
  HandlerThread backgroundThread = new HandlerThread("DepthCameraPreview");
  backgroundThread.start();
  Handler backgroundHandler = new Handler(backgroundThread.getLooper());
  
  try
  {
    depthCamPreviewCaptureSession.setRepeatingRequest(depthPreviewBuilder.build(), null, backgroundHandler);
  }
  catch (CameraAccessException e)
  {
    e.printStackTrace();
  }
}

@Override
public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession)
{
  Toast.makeText(mMainActivity, "Depth camera preview error!", Toast.LENGTH_LONG).show();
}

};

private ImageReader.OnImageAvailableListener depthImgReaderPreviewOnImageAvailable = new ImageReader.OnImageAvailableListener()
{
@Override
public void onImageAvailable(ImageReader imageReader)
{
try
{
Image image = imageReader.acquireNextImage();
if (image != null && image.getFormat() == ImageFormat.DEPTH16)
{
int grayScaleDepthImg = getGrayScaleDepthImg(image);
Bitmap grayScaleDepthBitmap = convertToARGBBitmap(grayScaleDepthImg);
grayScaleDepthBitmapG = grayScaleDepthBitmap;
renderBitmapToTextureView(grayScaleDepthBitmap, depthPreviewTextureView);
}
image.close();
}
catch (Exception e)
{
Toast.makeText(mMainActivity, "Failed to acquireNextImage: " + e.getMessage(), Toast.LENGTH_LONG).show();
}
}
};

private int getGrayScaleDepthImg(Image image)
{
int grayScaleDepthImg = new int[depthImgWidth*depthImgHeight];
ShortBuffer shortDepthBuffer = image.getPlanes()[0].getBuffer().asShortBuffer();

for (int i = 0; i < depthImgWidth*depthImgHeight; i++)
{
  int depth = shortDepthBuffer.get(i) & 0x1FFF;
  //depth = Math.max(200, depth);
  //depth = Math.min(1600, depth);
  grayScaleDepthImg[i] = depth;
}

Vector<Float> histograms = calculateHistogram(grayScaleDepthImg, Math.round((float)0xFFFF));

for (int i = 0; i < depthImgWidth*depthImgHeight; i++)
  grayScaleDepthImg[i] = histograms.get(grayScaleDepthImg[i]).intValue();

return grayScaleDepthImg;

}

Vector calculateHistogram(int depthImg, int histogramSize)
{
float numPts = 0;
Vector histograms = new Vector();
for (int i = 0; i < histogramSize; i++)
histograms.add(0.0f);

for (int i = 0; i < depthImgWidth*depthImgHeight; i++)
{
  if (depthImg[i] != 0)
  {
    histograms.set(depthImg[i], histograms.get(depthImg[i]) + 1);
    numPts++;
  }
}

for (int idx = 1; idx < histogramSize; idx++)
  histograms.set(idx, histograms.get(idx) + histograms.get(idx - 1));

if (numPts > 0)
  for (int idx = 1; idx < histogramSize; idx++)
    histograms.set(idx, (255.0f*(1.0f - (histograms.get(idx) / numPts))));
  
return histograms;

}

private Bitmap convertToARGBBitmap(int grayScaleDepthImg)
{
Bitmap bitmap = Bitmap.createBitmap(depthImgWidth, depthImgHeight, Bitmap.Config.ARGB_4444);
for (int y = 0; y < depthImgHeight; y++)
{
for (int x = 0; x < depthImgWidth; x++)
{
int index = y * depthImgWidth + x;
bitmap.setPixel(x, y, Color.argb(255, grayScaleDepthImg[index], grayScaleDepthImg[index], 0));
}
}
return bitmap;
}

private void renderBitmapToTextureView(Bitmap bitmap, TextureView textureView)
{
Bitmap bitmapTemp;
int displayRotation = windowMgr.getDefaultDisplay().getRotation();
if (displayRotation == Surface.ROTATION_0 || displayRotation == Surface.ROTATION_180)
{
Mat matTemp = new Mat();
Utils.bitmapToMat(bitmap, matTemp);
Core.transpose(matTemp, matTemp);
Core.flip(matTemp, matTemp, 1);
bitmapTemp = Bitmap.createBitmap(matTemp.width(), matTemp.height(), Bitmap.Config.ARGB_4444);
Utils.matToBitmap(matTemp, bitmapTemp);
matTemp.release();
}
else if (displayRotation == Surface.ROTATION_270)
{
Mat matTemp = new Mat();
Utils.bitmapToMat(bitmap, matTemp);
Core.flip(matTemp, matTemp, 0);
Core.flip(matTemp, matTemp, 1);
bitmapTemp = Bitmap.createBitmap(matTemp.width(), matTemp.height(), Bitmap.Config.ARGB_4444);
Utils.matToBitmap(matTemp, bitmapTemp);
matTemp.release();
}
else
bitmapTemp = bitmap;

Canvas canvas = textureView.lockCanvas();
if(canvas != null)
{
  canvas.drawBitmap(bitmapTemp, getBitmap2TextureViewTransform(textureView, bitmapTemp), null);
}
textureView.unlockCanvasAndPost(canvas);
bitmap.recycle();
bitmapTemp.recycle();

}

private Matrix getBitmap2TextureViewTransform(TextureView textureView, Bitmap bitmap)
{
Matrix matrix = new Matrix();
RectF textureViewRect = new RectF(0, 0, textureView.getWidth(), textureView.getHeight());
RectF bitmapRect = new RectF(0, 0, bitmap.getWidth(), bitmap.getHeight());
matrix.setRectToRect(bitmapRect, textureViewRect, Matrix.ScaleToFit.CENTER);
return matrix;
}

public void takePicture()
{
if(depthCameraDevice == null)
{
Toast.makeText(mMainActivity, “Depth Camera error!”, Toast.LENGTH_LONG).show();
return;
}

coloredDepthImgFile = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DCIM).getPath(), "coloredDepthImg.jpg");


try
{
  depthTakePicImgReader = ImageReader.newInstance(depthImgWidth, depthImgHeight, ImageFormat.DEPTH16, 2);

  HandlerThread thread = new HandlerThread(" DepthCameraTakePicture");
  thread.start();
  depthTakePicBGHandler = new Handler(thread.getLooper());

  depthTakePicImgReader.setOnImageAvailableListener(depthImgReaderTakePicOnImageAvailable, depthTakePicBGHandler);

  List<Surface> outputSurfaces = new ArrayList<Surface>(1);
  outputSurfaces.add(depthTakePicImgReader.getSurface());

  depthCapBuilder = depthCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
  depthCapBuilder.addTarget(depthTakePicImgReader.getSurface());
  depthCapBuilder.set(CaptureRequest.CONTROL_MODE,  CameraMetadata.CONTROL_MODE_AUTO);
  
  depthCameraDevice.createCaptureSession(outputSurfaces, depthCamTakePicCaptureSessionStateCallback, depthTakePicBGHandler);
}
catch (CameraAccessException e)
{
  e.printStackTrace();
}

}

ImageReader.OnImageAvailableListener depthImgReaderTakePicOnImageAvailable = new ImageReader.OnImageAvailableListener() // imagereader 有影像時，儲存影像
{
@Override
public void onImageAvailable(ImageReader imageReader)
{
Image image = null;

  try
  {
    image = imageReader.acquireNextImage();

    if (image != null && image.getFormat() == ImageFormat.DEPTH16)
    {
        int [] grayScaleDepthImg = getGrayScaleDepthImg_v2(image);
        Bitmap grayScaleDepthBitmap = convertToARGBBitmap(grayScaleDepthImg);
        renderBitmapToTextureView(grayScaleDepthBitmap, depthPreviewTextureView2);

      //Bitmap depthBitmap = convertToARGBBitmap(image);
      //renderBitmapToTextureView(depthBitmap, depthPreviewTextureView2);

// OutputStream output = null;
// try
// {
// output = new FileOutputStream(coloredDepthImgFile);
// output.write(stream.toByteArray());
// }
// finally
// {
// if(null != output)
// output.close();
// }
}
}
// catch (FileNotFoundException e)
// {
// e.printStackTrace();
// }
// catch (IOException e)
// {
// e.printStackTrace();
// }
finally
{
if(image != null)
image.close();
}
}
};

private int getGrayScaleDepthImg_v2(Image image)
{
int grayScaleDepthImg = new int[depthImgWidth*depthImgHeight];
ShortBuffer shortDepthBuffer = image.getPlanes()[0].getBuffer().asShortBuffer();

for (int i = 0; i < depthImgWidth*depthImgHeight; i++)
{
  int depth = shortDepthBuffer.get(i) & 0x1FFF;
  grayScaleDepthImg[i] = depth;
}

Vector<Float> histograms = calculateHistogram(grayScaleDepthImg, Math.round((float)0xFFFF));

for (int i = 0; i < depthImgWidth*depthImgHeight; i++)
  grayScaleDepthImg[i] = histograms.get(grayScaleDepthImg[i]).intValue();

return grayScaleDepthImg;

}

private Bitmap convertToARGBBitmap(Image image)
{
ByteBuffer colorBuffer = image.getPlanes()[0].getBuffer();
byte bytes = new byte[colorBuffer.capacity()];
colorBuffer.get(bytes);
Bitmap bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.length);
return bitmap;
}

private CameraCaptureSession.StateCallback depthCamTakePicCaptureSessionStateCallback = new CameraCaptureSession.StateCallback()
{
@Override
public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession)
{
try
{
closeAllCameraCaptureSession();

    depthCamTakePicCaptureSession = cameraCaptureSession;

    depthCamTakePicCaptureSession.capture(depthCapBuilder.build(), depthCamTakePicCaptureSessionCapCallback, depthTakePicBGHandler);
  }
  catch (CameraAccessException e)
  {
    e.printStackTrace();
  }
}

@Override
public void onConfigureFailed(@NonNull CameraCaptureSession cameraCaptureSession)
{
  Toast.makeText(mMainActivity, "Picture taking initialization error!", Toast.LENGTH_LONG).show();
}

};

CameraCaptureSession.CaptureCallback depthCamTakePicCaptureSessionCapCallback = new CameraCaptureSession.CaptureCallback()
{
@Override
public void onCaptureCompleted(@NonNull CameraCaptureSession session, @NonNull CaptureRequest request, @NonNull TotalCaptureResult result)
{
super.onCaptureCompleted(session, request, result);

  Uri uri = Uri.parse("android.resource://" + packageName + "/" + R.raw.sound_camera_shutter);
  MediaPlayer mp = MediaPlayer.create(mMainActivity, uri);
  mp.start();
  
  Toast.makeText(mMainActivity, "Finishing taking picture\nPicture: " + coloredDepthImgFile, Toast.LENGTH_SHORT).show();
  startDepthCamPreview();
}

@Override
public void onCaptureProgressed(CameraCaptureSession session, CaptureRequest request, CaptureResult partialResult) {}

};

private void closeAllCameraCaptureSession()
{
if(depthCamPreviewCaptureSession != null)
{
depthCamPreviewCaptureSession.close();
depthCamPreviewCaptureSession = null;
}

if(depthCamTakePicCaptureSession != null)
{
  depthCamTakePicCaptureSession.close();
  depthCamTakePicCaptureSession = null;
}

}

public void releaseCameraDevice()
{
if(depthCameraDevice != null)
{
depthCameraDevice.close();
depthCameraDevice = null;
}
}
}