Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Add environment separation for rigid objects #545

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

[Feature] Add environment separation for rigid objects #545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e5998d1
update LuisaCompute
ACMLCZH Dec 29, 2024
54180ba
Sync Luisa
ACMLCZH Dec 29, 2024
3251810
Add rigid separate
ACMLCZH Jan 7, 2025
c2e66fe
Add env idx
ACMLCZH Jan 8, 2025
3ee16d6
Merge branch 'main' of https://github.com/Genesis-Embodied-AI/Genesis…
ACMLCZH Jan 8, 2025
a1b2141
Merge branch 'main' of https://github.com/Genesis-Embodied-AI/Genesis…
ACMLCZH Jan 10, 2025
81be0bf
fix rasterizer normal env sep
ACMLCZH Jan 12, 2025
8323ba8
Merge branch 'main' of https://github.com/Genesis-Embodied-AI/Genesis…
ACMLCZH Jan 12, 2025
54194ae
update document
ACMLCZH Jan 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion genesis/ext/LuisaRender
Open in desktop
Submodule LuisaRender updated 7 files
+3 −2 .github/workflows/ci.yml
+2 −2 .gitmodules
+0 −1 src/base/pipeline.h
+1 −0 src/base/shape.h
+1 −1 src/compute
+1 −1 src/films/display.cpp
+1 −0 src/integrators/nfor.cpp
2 changes: 2 additions & 0 deletions genesis/ext/pyrender/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class RenderFlags(object):
"""Skip the floor"""
REFLECTIVE_FLOOR = 1 << 16
"""Render the reflection of floor"""
ENV_SEPARATE = 1 << 17
"""Render the reflection of floor"""


class TextAlign:
Expand Down
78 changes: 57 additions & 21 deletions genesis/ext/pyrender/jit_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,8 @@ def set_primitive(self, scene, node_list, primitive_list):
self.pbr_mat = np.zeros((n, 9), np.float32) # base_color <- 4, metallic <- 1, roughness <- 1, emissive <- 3
self.spec_mat = np.zeros((n, 11), np.float32) # diffuse <- 4, specular <- 3, glossiness <- 1, emissive <- 3
self.render_flags = np.zeros(
(n, 6), np.int8
) # (blend, wireframe, double sided, pbr texture, reflective floor, transparent)
(n, 7), np.int8
) # (blend, wireframe, double sided, pbr texture, reflective floor, transparent, env shared)
self.mode = np.zeros(n, np.int32)
self.n_instances = np.zeros(n, np.int32)
self.n_indices = np.zeros(n, np.int32) # positive: indices, negative: positions
Expand Down Expand Up @@ -334,6 +334,7 @@ def set_primitive(self, scene, node_list, primitive_list):
self.render_flags[i, 3] = isinstance(material, MetallicRoughnessMaterial)
self.render_flags[i, 4] = primitive.is_floor and not floor_existed
self.render_flags[i, 5] = node_list[i].mesh.is_transparent
self.render_flags[i, 6] = primitive.env_shared

if primitive.is_floor:
floor_existed = True
Expand Down Expand Up @@ -377,6 +378,7 @@ def gen_func_ptr(self):
float32[:, :],
int32,
float32[:],
int32,
self.gl.wrapper_type,
),
cache=True,
Expand Down Expand Up @@ -404,6 +406,7 @@ def forward_pass(
reflection_mat,
floor_tex,
screen_size,
env_idx,
gl,
):
det_reflection = np.linalg.det(reflection_mat)
Expand Down Expand Up @@ -508,12 +511,20 @@ def forward_pass(
continue
set_uniform_3fv(pid, "color", color_list[id], gl)

if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
if render_flags[id, 6] or env_idx == -1:
if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
if n_indices[id] > 0:
gl.glDrawElementsInstancedBaseInstance(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), 1, env_idx
)
else:
gl.glDrawArraysInstancedBaseInstance(mode[id], 0, -n_indices[id], 1, env_idx)

gl.glBindVertexArray(0)
gl.glUseProgram(0)
Expand All @@ -530,11 +541,14 @@ def forward_pass(
float32[:, :],
float32[:, :],
int8[:, :],
int32,
self.gl.wrapper_type,
),
cache=True,
)
def shadow_mapping_pass(vao_id, program_id, pose, mode, n_instances, n_indices, mat_V, mat_P, render_flags, gl):
def shadow_mapping_pass(
vao_id, program_id, pose, mode, n_instances, n_indices, mat_V, mat_P, render_flags, env_idx, gl
):
last_pid = -1
for id in range(len(vao_id)):
if render_flags[id, 5]:
Expand All @@ -559,12 +573,20 @@ def shadow_mapping_pass(vao_id, program_id, pose, mode, n_instances, n_indices,

gl.glDisable(GL_PROGRAM_POINT_SIZE)

if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
if render_flags[id, 6] or env_idx == -1:
if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
if n_indices[id] > 0:
gl.glDrawElementsInstancedBaseInstance(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), 1, env_idx
)
else:
gl.glDrawArraysInstancedBaseInstance(mode[id], 0, -n_indices[id], 1, env_idx)

gl.glBindVertexArray(0)
gl.glUseProgram(0)
Expand All @@ -581,12 +603,13 @@ def shadow_mapping_pass(vao_id, program_id, pose, mode, n_instances, n_indices,
float32[:, :, :],
float32[:],
int8[:, :],
int32,
self.gl.wrapper_type,
),
cache=True,
)
def point_shadow_mapping_pass(
vao_id, program_id, pose, mode, n_instances, n_indices, light_matrix, light_pos, render_flags, gl
vao_id, program_id, pose, mode, n_instances, n_indices, light_matrix, light_pos, render_flags, env_idx, gl
):
last_pid = -1
for id in range(len(vao_id)):
Expand All @@ -613,12 +636,20 @@ def point_shadow_mapping_pass(

gl.glDisable(GL_PROGRAM_POINT_SIZE)

if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
if render_flags[id, 6] or env_idx == -1:
if n_indices[id] > 0:
gl.glDrawElementsInstanced(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), n_instances[id]
)
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
else:
gl.glDrawArraysInstanced(mode[id], 0, -n_indices[id], n_instances[id])
if n_indices[id] > 0:
gl.glDrawElementsInstancedBaseInstance(
mode[id], n_indices[id], GL_UNSIGNED_INT, address_to_ptr(0), 1, env_idx
)
else:
gl.glDrawArraysInstancedBaseInstance(mode[id], 0, -n_indices[id], 1, env_idx)

gl.glBindVertexArray(0)
gl.glUseProgram(0)
Expand Down Expand Up @@ -704,6 +735,7 @@ def forward_pass(
color_list=None,
reflection_mat=np.identity(4, np.float32),
floor_tex=0,
env_idx=-1,
):
self.load_programs(renderer, flags, program_flags)
func = self._forward_pass
Expand Down Expand Up @@ -732,6 +764,7 @@ def forward_pass(
reflection_mat,
floor_tex,
screen_size,
env_idx,
self.gl.wrapper_instance,
)
else:
Expand All @@ -758,11 +791,12 @@ def forward_pass(
reflection_mat,
floor_tex,
screen_size,
env_idx,
self.gl.wrapper_instance,
)
# print(100.0/(time()-timer))

def shadow_mapping_pass(self, renderer, V, P, flags, program_flags):
def shadow_mapping_pass(self, renderer, V, P, flags, program_flags, env_idx=-1):
self.load_programs(renderer, flags, program_flags)
func = self._shadow_mapping_pass
func(
Expand All @@ -775,10 +809,11 @@ def shadow_mapping_pass(self, renderer, V, P, flags, program_flags):
V.astype(np.float32),
P.astype(np.float32),
self.render_flags,
env_idx,
self.gl.wrapper_instance,
)

def point_shadow_mapping_pass(self, renderer, light_matrix, light_pos, flags, program_flags):
def point_shadow_mapping_pass(self, renderer, light_matrix, light_pos, flags, program_flags, env_idx=-1):
self.load_programs(renderer, flags, program_flags)
func = self._point_shadow_mapping_pass
func(
Expand All @@ -791,6 +826,7 @@ def point_shadow_mapping_pass(self, renderer, light_matrix, light_pos, flags, pr
light_matrix.astype(np.float32),
light_pos.astype(np.float32),
self.render_flags,
env_idx,
self.gl.wrapper_instance,
)

Expand Down
4 changes: 3 additions & 1 deletion genesis/ext/pyrender/mesh.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ def from_trimesh(
smooth=False,
double_sided=False,
is_floor=False,
env_shared=True,
):
"""Create a Mesh from a :class:`~trimesh.base.Trimesh`.

Expand All @@ -162,7 +163,7 @@ def from_trimesh(
If not specified and the mesh has no material, a default material
will be used.
is_visible : bool
If False, the mesh will not be rendered.
If `False`, the mesh will not be rendered.
poses : (n,4,4) float
Array of 4x4 transformation matrices for instancing this object.
wireframe : bool
Expand Down Expand Up @@ -231,6 +232,7 @@ def from_trimesh(
vertex_mapping=vertex_mapping,
double_sided=double_sided,
is_floor=is_floor,
env_shared=env_shared,
)
)

Expand Down
2 changes: 2 additions & 0 deletions genesis/ext/pyrender/numba_gl_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ def __init__(self):
load_func("glCullFace", GLvoid, GLenum)
load_func("glDrawElementsInstanced", GLvoid, GLenum, GLsizei, GLenum, GLvoidp, GLsizei)
load_func("glDrawArraysInstanced", GLvoid, GLenum, GLint, GLsizei, GLsizei)
load_func("glDrawElementsInstancedBaseInstance", GLvoid, GLenum, GLsizei, GLenum, GLvoidp, GLsizei, GLuint)
load_func("glDrawArraysInstancedBaseInstance", GLvoid, GLenum, GLint, GLsizei, GLsizei, GLuint)
load_func("glUseProgram", GLvoid, GLuint)
load_func("glFlush", GLvoid)
load_func("glReadPixels", GLvoid, GLint, GLint, GLsizei, GLsizei, GLenum, GLenum, GLvoidp)
Expand Down
11 changes: 10 additions & 1 deletion genesis/ext/pyrender/offscreen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def render(
seg=False,
ret_depth=False,
plane_reflection=False,
env_separate_rigid=False,
normal=False,
):
"""Render a scene with the given set of flags.
Expand Down Expand Up @@ -112,6 +113,9 @@ def render(
if plane_reflection:
flags |= RenderFlags.REFLECTIVE_FLOOR

if env_separate_rigid:
flags |= RenderFlags.ENV_SEPARATE

if seg:
seg_node_map = self._seg_node_map
flags |= RenderFlags.SEG
Expand Down Expand Up @@ -151,7 +155,12 @@ def get_program(self, vertex_shader, fragment_shader, geometry_shader=None, defi

old_cache = renderer._program_cache
renderer._program_cache = CustomShaderCache()
normal_arr, _ = renderer.render(scene, RenderFlags.FLAT | RenderFlags.OFFSCREEN)

flags = RenderFlags.FLAT | RenderFlags.OFFSCREEN
if env_separate_rigid:
flags |= RenderFlags.ENV_SEPARATE

normal_arr, _ = renderer.render(scene, flags)
retval = retval + (normal_arr,)
renderer._program_cache = old_cache

Expand Down
2 changes: 2 additions & 0 deletions genesis/ext/pyrender/primitive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def __init__(
vertex_mapping=None,
double_sided=False,
is_floor=False,
env_shared=True,
):

if mode is None:
Expand All @@ -95,6 +96,7 @@ def __init__(
self.vertex_mapping = vertex_mapping
self.double_sided = double_sided
self.is_floor = is_floor
self.env_shared = env_shared

self._bounds = None
self._vaid = None
Expand Down
Loading