· 2 years ago · Apr 15, 2023, 01:10 PM
1using System.Collections;
2using System.Collections.Generic;
3using UnityEngine;
4using com.rfilkov.kinect;
5
6
7namespace com.rfilkov.components
8{
9 /// <summary>
10 /// SceneMeshRendererGpu renders virtual mesh of the environment in the scene, as detected by the given sensor. This component uses GPU for mesh processing rather than CPU.
11 /// </summary>
12 public class SceneMeshRendererGpu : MonoBehaviour
13 {
14 private MeshFilter meshFilter;
15
16 void StartMethod()
17 {
18 // get reference to mesh filter
19 meshFilter = GetComponent<MeshFilter>();
20 }
21 [Tooltip("Depth sensor index - 0 is the 1st one, 1 - the 2nd one, etc.")]
22 public int sensorIndex = 0;
23
24 [Tooltip("Resolution of the images used to generate the scene.")]
25 private DepthSensorBase.PointCloudResolution sourceImageResolution = DepthSensorBase.PointCloudResolution.DepthCameraResolution;
26
27 [Tooltip("Whether to show the mesh as point cloud, or as solid mesh.")]
28 public bool showAsPointCloud = true;
29
30 [Tooltip("Mesh coarse factor.")]
31 [Range(1, 4)]
32 public int coarseFactor = 1;
33
34 [Tooltip("Edge cut-off factor.")]
35 [Range(0f, 1f)]
36 public float edgeCutoffFactor = 0.5f;
37
38 [Tooltip("Whether the mesh texture is RGB or infrared.")]
39 public MeshTextureType meshTexture = MeshTextureType.ColorTexture;
40 public enum MeshTextureType : int { ColorTexture = 0, InfraredTexture = 1 }
41
42
43 [Tooltip("Horizontal limit - minimum, in meters.")]
44 [Range(-5f, 5f)]
45 public float xMin = -2f;
46
47 [Tooltip("Horizontal limit - maximum, in meters.")]
48 [Range(-5f, 5f)]
49 public float xMax = 2f;
50
51 [Tooltip("Vertical limit - minimum, in meters.")]
52 [Range(-5f, 5f)]
53 public float yMin = 0f;
54
55 [Tooltip("Vertical limit - maximum, in meters.")]
56 [Range(-5f, 5f)]
57 public float yMax = 2.5f;
58
59 [Tooltip("Distance limit - minimum, in meters.")]
60 [Range(0.5f, 10f)]
61 public float zMin = 1f;
62
63 [Tooltip("Distance limit - maximum, in meters.")]
64 [Range(0.5f, 10f)]
65 public float zMax = 3f;
66
67 [Tooltip("Time interval between scene mesh updates, in seconds. 0 means no wait.")]
68 private float updateMeshInterval = 0f;
69
70 [Tooltip("Time interval between mesh-collider updates, in seconds. 0 means no mesh-collider updates.")]
71 private float updateColliderInterval = 0f;
72
73 [Tooltip("Whether to apply lighting to the mesh, or not.")]
74 public bool applyLighting = false;
75
76
77 // reference to object's mesh
78 private Mesh mesh = null;
79 private Transform trans = null;
80
81 // references to KM and data
82 private KinectManager kinectManager = null;
83 private KinectInterop.SensorData sensorData = null;
84 private DepthSensorBase sensorInt = null;
85 //private Vector3 spaceScale = Vector3.zero;
86 private Material meshShaderMat = null;
87
88 // space table
89 private Vector3[] spaceTable = null;
90 private ComputeBuffer spaceTableBuffer = null;
91 private Vector3 sensorSpaceScale = Vector3.zero;
92
93 private ushort[] depthImageCopy = null;
94 private ComputeBuffer depthImageBuffer = null;
95
96 // render textures
97 private RenderTexture colorTexture = null;
98 private RenderTexture colorTextureCopy = null;
99 private bool colorTextureCreated = false;
100 //private bool depthBufferCreated = false;
101
102 // times
103 private ulong lastDepthFrameTime = 0;
104 private float lastMeshUpdateTime = 0f;
105 private float lastColliderUpdateTime = 0f;
106
107 // image parameters
108 private int imageWidth = 0;
109 private int imageHeight = 0;
110
111 // mesh parameters
112 private bool bMeshInited = false;
113 private int meshParamsCache = 0;
114
115 // lighting
116 private FragmentLighting lighting = new FragmentLighting();
117
118 public Mesh GetMesh()
119 {
120 // Get the MeshFilter component attached to this GameObject
121 MeshFilter meshFilter = GetComponent<MeshFilter>();
122
123 // Check if the MeshFilter component exists
124 if (meshFilter == null)
125 {
126 Debug.LogError("No MeshFilter component found!");
127 return null;
128 }
129
130 // Get the mesh from the MeshFilter component
131 Mesh mesh = meshFilter.mesh;
132
133 return mesh;
134 }
135
136
137 void Start()
138 {
139 // get sensor data
140 kinectManager = KinectManager.Instance;
141 sensorData = (kinectManager != null && kinectManager.IsInitialized()) ? kinectManager.GetSensorData(sensorIndex) : null;
142
143 if (meshTexture == MeshTextureType.InfraredTexture && kinectManager && kinectManager.GetInfraredImageTex(sensorIndex) == null)
144 {
145 Debug.LogError("Please set the 'Get Infrared Frames'-setting of KinectManager to 'Infrared texture'.");
146 }
147
148 // find scene lights
149 Light[] sceneLights = GameObject.FindObjectsOfType<Light>();
150 lighting.SetLightsAndBounds(sceneLights, transform.position, new Vector3(20f, 20f, 20f));
151 }
152
153
154 void OnDestroy()
155 {
156 if (bMeshInited)
157 {
158 // release the mesh-related resources
159 FinishMesh();
160 }
161
162 // release lighting resources
163 lighting.ReleaseResources();
164 }
165
166
167 void Update()
168 {
169 if (mesh == null && sensorData != null && sensorData.depthCamIntr != null)
170 {
171 // init mesh and its related data
172 InitMesh();
173 }
174
175 if (bMeshInited)
176 {
177 // update the mesh
178 UpdateMesh();
179 }
180 }
181
182
183 // inits the mesh and related data
184 private void InitMesh()
185 {
186 // create mesh
187 mesh = new Mesh
188 {
189 name = "SceneMesh-Sensor" + sensorIndex,
190 indexFormat = UnityEngine.Rendering.IndexFormat.UInt32
191 };
192
193 MeshFilter meshFilter = GetComponent<MeshFilter>();
194 if (meshFilter != null)
195 {
196 meshFilter.mesh = mesh;
197 }
198 else
199 {
200 Debug.LogWarning("MeshFilter not found! You may not see the mesh on screen");
201 }
202
203 // get the mesh material
204 Renderer meshRenderer = GetComponent<Renderer>();
205 if (meshRenderer)
206 {
207 Shader meshShader = Shader.Find("Kinect/SceneMeshUShader");
208 if(meshShader != null)
209 {
210 meshShaderMat = new Material(meshShader);
211 meshRenderer.material = meshShaderMat;
212 }
213 }
214
215 // get reference to the transform
216 trans = GetComponent<Transform>();
217
218 // get sensor interface
219 sensorInt = sensorData != null ? (DepthSensorBase)sensorData.sensorInterface : null;
220
221 // create point cloud color texture
222 if (sensorData != null && sensorInt != null && meshShaderMat != null)
223 {
224 Vector2Int imageRes = Vector2Int.zero;
225 if (sensorInt.pointCloudColorTexture == null)
226 {
227 sensorInt.pointCloudResolution = sourceImageResolution;
228 imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
229
230 //colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
231 //sensorInt.pointCloudColorTexture = colorTexture;
232 colorTextureCreated = true;
233 }
234 else
235 {
236 sourceImageResolution = sensorInt.pointCloudResolution;
237 imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
238 colorTexture = sensorInt.pointCloudColorTexture;
239 colorTextureCreated = false;
240 }
241
242 // get space scale
243 sensorSpaceScale = kinectManager.GetSensorSpaceScale(sensorIndex); // kinectManager.GetDepthImageScale(sensorIndex)
244
245 // update textures and buffers
246 UpdateTexturesAndBuffers();
247
248 bMeshInited = true;
249 }
250 }
251
252
253 // updates the textures and buffers, needed by the mesh renderer
254 private void UpdateTexturesAndBuffers()
255 {
256 if (sensorData == null || sensorInt == null)
257 return;
258
259 // get image resolution
260 Vector2Int imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
261
262 if (colorTextureCreated && (colorTexture == null ||
263 colorTexture.width != imageRes.x || colorTexture.height != imageRes.y))
264 {
265 colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
266 sensorInt.pointCloudColorTexture = colorTexture;
267 //Debug.Log("Created pointCloudColorTexture with resolution " + imageRes);
268 }
269
270 // create depth image buffer
271 if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
272 {
273 int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight >> 1;
274
275 if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength)
276 {
277 depthImageCopy = new ushort[depthBufferLength << 1];
278 depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint));
279 //Debug.Log("Created depthImageBuffer with length " + depthBufferLength);
280 }
281
282 //meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer);
283 }
284 else
285 {
286 int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight >> 1;
287
288 if (sensorData.colorDepthBuffer == null || sensorData.colorDepthBuffer.count != bufferLength)
289 {
290 sensorData.colorDepthBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorDepthBuffer, bufferLength, sizeof(uint));
291 //Debug.Log("Created colorDepthBuffer with length " + bufferLength);
292 }
293
294 //meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer);
295
296 if (meshTexture == MeshTextureType.InfraredTexture)
297 {
298 if (sensorData.colorInfraredBuffer == null || sensorData.colorInfraredBuffer.count != bufferLength)
299 {
300 sensorData.colorInfraredBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorInfraredBuffer, bufferLength, sizeof(uint));
301 //Debug.Log("Created colorInfraredBuffer with length " + bufferLength);
302 }
303 }
304 }
305
306 if (colorTextureCopy == null || colorTextureCopy.width != imageRes.x || colorTextureCopy.height != imageRes.y)
307 {
308 // create copy texture
309 colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
310 //Debug.Log("Created colorTextureCopy with resolution " + imageRes);
311 }
312
313 if (spaceTableBuffer == null || imageWidth != imageRes.x || imageHeight != imageRes.y)
314 {
315 // create space table
316 spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ?
317 sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData);
318
319 int spaceBufferLength = imageRes.x * imageRes.y * 3;
320 spaceTableBuffer = KinectInterop.CreateComputeBuffer(spaceTableBuffer, spaceBufferLength, sizeof(float));
321 spaceTableBuffer.SetData(spaceTable);
322 spaceTable = null;
323
324 //Debug.Log("Created spaceTable for resolution " + imageRes);
325 }
326
327 // create mesh vertices & indices
328 if (imageWidth != imageRes.x || imageHeight != imageRes.y)
329 {
330 CreateMeshVertInd(imageRes.x, imageRes.y);
331 }
332
333 // image width & height
334 imageWidth = imageRes.x;
335 imageHeight = imageRes.y;
336 }
337
338
339 // creates the mesh vertices and indices
340 private void CreateMeshVertInd(int imageWidth, int imageHeight)
341 {
342 int xVerts = (imageWidth / coarseFactor); // + 1;
343 int yVerts = (imageHeight / coarseFactor); // + 1;
344 int vCount = xVerts * yVerts;
345 //Debug.Log("xVerts: " + xVerts + ", yVerts: " + yVerts + ", vCount: " + vCount);
346
347 // mesh vertices
348 mesh.Clear();
349 mesh.indexFormat = UnityEngine.Rendering.IndexFormat.UInt32;
350
351 Vector3[] meshVertices = new Vector3[vCount];
352 Vector3[] meshNormals = new Vector3[vCount];
353 Vector2[] meshUv = new Vector2[vCount];
354
355 float vsx = (float)coarseFactor / (float)imageWidth;
356 float vsy = (float)coarseFactor / (float)imageHeight;
357
358 for (int y = 0, vi = 0; y < yVerts; y++)
359 {
360 for (int x = 0; x < xVerts; x++, vi++)
361 {
362 meshVertices[vi] = new Vector3(x * vsx, y * vsy, 0f);
363 meshNormals[vi] = new Vector3(0f, 1f, 0f); // 0f, 0f, -1f
364 meshUv[vi] = new Vector2(x * vsx, y * vsy);
365 }
366 }
367
368 mesh.vertices = meshVertices;
369 mesh.normals = meshNormals;
370 mesh.uv = meshUv;
371
372 // mesh indices
373 if (showAsPointCloud)
374 {
375 int[] meshIndices = new int[vCount];
376 for (int i = 0; i < vCount; i++)
377 meshIndices[i] = i;
378
379 mesh.SetIndices(meshIndices, MeshTopology.Points, 0);
380 }
381 else
382 {
383 int[] meshIndices = new int[(xVerts - 1) * (yVerts - 1) * 6];
384 for (int y = 0, ii = 0; y < (yVerts - 1); y++)
385 {
386 for (int x = 0; x < (xVerts - 1); x++)
387 {
388 meshIndices[ii++] = (y + 1) * xVerts + x;
389 meshIndices[ii++] = y * xVerts + x + 1;
390 meshIndices[ii++] = y * xVerts + x;
391
392 meshIndices[ii++] = (y + 1) * xVerts + x + 1;
393 meshIndices[ii++] = y * xVerts + x + 1;
394 meshIndices[ii++] = (y + 1) * xVerts + x;
395 }
396 }
397
398 mesh.SetIndices(meshIndices, MeshTopology.Triangles, 0);
399 }
400
401 meshParamsCache = coarseFactor + (showAsPointCloud ? 10 : 0);
402 }
403
404
405 // releases mesh-related resources
406 private void FinishMesh()
407 {
408 if (sensorInt)
409 {
410 sensorInt.pointCloudColorTexture = null;
411 }
412
413 if(depthImageBuffer != null /**&& depthBufferCreated*/)
414 {
415 depthImageCopy = null;
416
417 depthImageBuffer.Release();
418 depthImageBuffer.Dispose();
419 depthImageBuffer = null;
420 }
421
422 if (sensorData.colorDepthBuffer != null /**&& depthBufferCreated*/)
423 {
424 sensorData.colorDepthBuffer.Release();
425 sensorData.colorDepthBuffer.Dispose();
426 sensorData.colorDepthBuffer = null;
427 }
428
429 if(sensorData.colorInfraredBuffer != null)
430 {
431 sensorData.colorInfraredBuffer.Release();
432 sensorData.colorInfraredBuffer.Dispose();
433 sensorData.colorInfraredBuffer = null;
434 }
435
436 if (colorTexture && colorTextureCreated)
437 {
438 colorTexture.Release();
439 colorTexture = null;
440 }
441
442 if (colorTextureCopy)
443 {
444 colorTextureCopy.Release();
445 colorTextureCopy = null;
446 }
447
448 if (spaceTableBuffer != null)
449 {
450 spaceTableBuffer.Dispose();
451 spaceTableBuffer = null;
452 }
453
454 bMeshInited = false;
455 }
456
457
458 // updates the mesh according to current depth frame
459 private void UpdateMesh()
460 {
461 if (bMeshInited && sensorData.depthImage != null && sensorData.depthCamIntr != null && meshShaderMat != null &&
462 lastDepthFrameTime != sensorData.lastDepthFrameTime && (Time.time - lastMeshUpdateTime) >= updateMeshInterval)
463 {
464 lastDepthFrameTime = sensorData.lastDepthFrameTime;
465 lastMeshUpdateTime = Time.time;
466
467 // update textures and buffers
468 UpdateTexturesAndBuffers();
469
470 int paramsCache = coarseFactor + (showAsPointCloud ? 10 : 0);
471 if(meshParamsCache != paramsCache)
472 {
473 //Debug.Log("Mesh params changed. Recreating...");
474 CreateMeshVertInd(imageWidth, imageHeight);
475 }
476
477 if (depthImageBuffer != null && sensorData.depthImage != null /**&& depthBufferCreated*/)
478 {
479 KinectInterop.CopyBytes(sensorData.depthImage, sizeof(ushort), depthImageCopy, sizeof(ushort));
480
481 int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2;
482 KinectInterop.SetComputeBufferData(depthImageBuffer, depthImageCopy, depthBufferLength, sizeof(uint));
483 }
484
485 switch(meshTexture)
486 {
487 case MeshTextureType.ColorTexture:
488 if(colorTexture != null)
489 Graphics.CopyTexture(colorTexture, colorTextureCopy);
490 break;
491
492 case MeshTextureType.InfraredTexture:
493 Texture infraredTexture = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ?
494 sensorData.infraredImageTexture : sensorData.colorInfraredTexture;
495 if(infraredTexture != null)
496 Graphics.CopyTexture(infraredTexture, colorTextureCopy);
497 break;
498 }
499
500 if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
501 {
502 //Debug.Log("SceneMeshGpu DepthFrameTime: " + lastDepthFrameTime);
503 }
504 else
505 {
506 //sensorData.usedColorDepthBufferTime = sensorData.lastColorDepthBufferTime;
507 //Debug.Log("SceneMeshGpu ColorDepthBufferTime: " + sensorData.lastColorDepthBufferTime);
508 }
509
510 if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
511 meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer);
512 else
513 meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer);
514
515 meshShaderMat.SetTexture("_ColorTex", colorTextureCopy);
516 meshShaderMat.SetVector("_SpaceScale", sensorSpaceScale);
517 meshShaderMat.SetBuffer("_SpaceTable", spaceTableBuffer);
518
519 meshShaderMat.SetVector("_TexRes", new Vector2(imageWidth, imageHeight));
520 //meshShaderMat.SetVector("_Cxy", new Vector2(sensorData.depthCamIntr.ppx, sensorData.depthCamIntr.ppy));
521 //meshShaderMat.SetVector("_Fxy", new Vector2(sensorData.depthCamIntr.fx, sensorData.depthCamIntr.fy));
522
523 meshShaderMat.SetInt("_CoarseFactor", coarseFactor);
524 meshShaderMat.SetInt("_IsPointCloud", showAsPointCloud ? 1 : 0);
525 meshShaderMat.SetFloat("_CutoffFactor", Mathf.Pow(edgeCutoffFactor, 6));
526
527 meshShaderMat.SetMatrix("_Transform", sensorInt.GetSensorToWorldMatrix());
528 meshShaderMat.SetVector("_PosMin", new Vector3(xMin, yMin, zMin));
529 meshShaderMat.SetVector("_PosMax", new Vector3(xMax, yMax, zMax));
530
531 // mesh bounds
532 Vector3 boundsCenter = new Vector3((xMax - xMin) / 2f, (yMax - yMin) / 2f, (zMax /**- zMin*/) / 2f);
533 Vector3 boundsSize = new Vector3((xMax - xMin), (yMax - yMin), (zMax /**- zMin*/));
534 mesh.bounds = new Bounds(boundsCenter, boundsSize);
535
536 // update lighting parameters
537 lighting.UpdateLighting(meshShaderMat, applyLighting);
538
539 if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval)
540 {
541 lastColliderUpdateTime = Time.time;
542 MeshCollider meshCollider = GetComponent<MeshCollider>();
543
544 if (meshCollider)
545 {
546 meshCollider.sharedMesh = null;
547 meshCollider.sharedMesh = mesh;
548 }
549 }
550
551 }
552 }
553
554 }
555}
556