From 4b7e7fd230b219d901809491cb7d922964830b1e Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 12 Sep 2024 16:22:56 +0800 Subject: [PATCH 01/23] Added .gitignore for sanity. --- .gitignore | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58016b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# vim swaps +.*.sw? + +# python binaries +*.py[oc] + +# numpy arrays +*.np[yz] + +# HumanML3D texts, when unzipped +HumanML3D/texts/*.txt + +# zip files +*.zip +*.bz2 +*.tar +*.gz +*.tar.gz +*.tgz From 998a550f43bf5ad18a36a446dd224650c4c0f0b3 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 12 Sep 2024 16:24:33 +0800 Subject: [PATCH 02/23] Updated raw pose processing for local machine. --- raw_pose_processing.ipynb | 65 +++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/raw_pose_processing.ipynb b/raw_pose_processing.ipynb index 6c94871..6f6b468 100644 --- a/raw_pose_processing.ipynb +++ b/raw_pose_processing.ipynb @@ -36,34 +36,37 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Please remember to download the following subdataset from AMASS website: https://amass.is.tue.mpg.de/download.php. Note only download the SMPL+H G data.\n", - "* ACCD (ACCD)\n", - "* HDM05 (MPI_HDM05)\n", - "* TCDHands (TCD_handMocap)\n", - "* SFU (SFU)\n", - "* BMLmovi (BMLmovi)\n", - "* CMU (CMU)\n", - "* Mosh (MPI_mosh)\n", - "* EKUT (EKUT)\n", - "* KIT (KIT)\n", - "* Eyes_Janpan_Dataset (Eyes_Janpan_Dataset)\n", - "* BMLhandball (BMLhandball)\n", - "* Transitions (Transitions_mocap)\n", - "* PosePrior (MPI_Limits)\n", - "* HumanEva (HumanEva)\n", - "* SSM (SSM_synced)\n", - "* DFaust (DFaust_67)\n", - "* TotalCapture (TotalCapture)\n", - "* BMLrub (BioMotionLab_NTroje)\n", + "### Please remember to download the following sub-datasets from AMASS website: https://amass.is.tue.mpg.de/download.php. Note only download the SMPL+H G data.\n", "\n", - "### Unzip all datasets. In the bracket we give the name of the unzipped file folder. Please correct yours to the given names if they are not the same." + "| Model Name | Folder Name | License | Data Size |\n", + "|--------------------|---------------------|-----------------|-----------|\n", + "| ACCAD | ACCAD | CC | 104 MB |\n", + "| BMLhandball | BMLhandball | BML | 398 MB |\n", + "| BMLmovi | BMLmovi | BML | 714 MB |\n", + "| BMLrub | BioMotionLab_NTroje | BML | 2.0 GB |\n", + "| CMU | CMU | CMU | 1.9 GB |\n", + "| DFaust | DFaust_67 | DFaust | 82.4 MB |\n", + "| EKUT | EKUT | KIT | 100 MB |\n", + "| Eyes_Japan_Dataset | Eyes_Japan_Dataset | Attribution 2.1 | 1.3 GB |\n", + "| HDM05 | MPI_HDM05 | CC | 501 MB |\n", + "| HumanEva | HumanEva | Custom | 31 MB |\n", + "| KIT | KIT | KITML | 2.0 GB |\n", + "| Mosh | MPI_mosh | MPI | 58.9 MB |\n", + "| PosePrior | MPI_Limits | MPI | 80.8 MB |\n", + "| SFU | SFU | SFU | 59.4 MB |\n", + "| SSM | SSM_synced | MPI | 12.9 MB |\n", + "| TCDHands | TCD_handMocap | TCD | 72.0 MB |\n", + "| TotalCapture | TotalCapture | TotalCap | 80.1 MB |\n", + "| Transitions | Transitions_mocap | MPI | 125 MB |\n", + "\n", + "### Unzip all datasets. You may need to install software that handles the bzip2 format. In the bracket we give the name of the unzipped file folder. Please correct yours to the given names if they are not the same." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Place all files under the directory **./amass_data/**. The directory structure shoud look like the following: \n", + "### Place all files under the directory **./amass_data/**. The directory structure should look like the following: \n", "./amass_data/ \n", "./amass_data/ACCAD/ \n", "./amass_data/BioMotionLab_NTroje/ \n", @@ -84,7 +87,7 @@ "./amass_data/TotalCapture/ \n", "./amass_data/Transitions_mocap/ \n", "\n", - "**Please make sure the file path are correct, otherwise it can not succeed.**" + "**Please make sure the file paths are correct, otherwise it can not succeed.**" ] }, { @@ -94,7 +97,7 @@ "outputs": [], "source": [ "# Choose the device to run the body model on.\n", - "comp_device = torch.device(\"cuda:2\" if torch.cuda.is_available() else \"cpu\")" + "comp_device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" ] }, { @@ -254,8 +257,10 @@ " for path in pbar:\n", " save_path = path.replace('./amass_data', './pose_data')\n", " save_path = save_path[:-3] + 'npy'\n", - " fps = amass_to_pose(path, save_path)\n", - " \n", + " try:\n", + " fps = amass_to_pose(path, save_path)\n", + " except OSError as oerr:\n", + " print(oerr) # for random non-pickle files in data\n", " cur_count += len(paths)\n", " print('Processed / All (fps %d): %d/%d'% (fps, cur_count, all_count) )\n", " time.sleep(0.5)" @@ -284,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -326,7 +331,7 @@ "outputs": [], "source": [ "index_path = './index.csv'\n", - "save_dir = './joints'\n", + "save_dir = './joints' # for some reason it does not exist by default?\n", "index_file = pd.read_csv(index_path)\n", "total_amount = index_file.shape[0]\n", "fps = 20" @@ -382,9 +387,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:torch_render]", + "display_name": "torch_render", "language": "python", - "name": "conda-env-torch_render-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -396,7 +401,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.19" } }, "nbformat": 4, From c5bf04ff5d258e30a887c3341eaf72408e4c27f1 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 18 Sep 2024 12:00:33 +0800 Subject: [PATCH 03/23] Updated README.md. --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8643f0d..4ddaa47 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,16 @@ We double the size of HumanML3D dataset by mirroring all motions and properly re [KIT Motion-Language Dataset](https://motion-annotation.humanoids.kit.edu/dataset/) (KIT-ML) is also a related dataset that contains 3,911 motions and 6,278 descriptions. We processed KIT-ML dataset following the same procedures of HumanML3D dataset, and provide the access in this repository. However, if you would like to use KIT-ML dataset, please remember to cite the original paper. -If this dataset is usefule in your projects, we will apprecite your star on this codebase. 😆😆 -## Checkout Our Works on HumanML3D +If this dataset is useful in your projects, we will appreciate your star on this codebase. 😆😆 + +## Checkout Our Work on HumanML3D :ok_woman: [T2M](https://ericguo5513.github.io/text-to-motion) - The first work on HumanML3D that learns to generate 3D motion from textual descriptions, with *temporal VAE*. :running: [TM2T](https://ericguo5513.github.io/TM2T) - Learns the mutual mapping between texts and motions through the discrete motion token. :dancer: [TM2D](https://garfield-kh.github.io/TM2D/) - Generates dance motions with text instruction. :honeybee: [MoMask](https://ericguo5513.github.io/momask/) - New-level text2motion generation using residual VQ and generative masked modeling. ## How to Obtain the Data -For KIT-ML dataset, you could directly download [[Here]](https://drive.google.com/drive/folders/1D3bf2G2o4Hv-Ale26YW18r1Wrh7oIAwK?usp=sharing). Due to the distribution policy of AMASS dataset, we are not allowed to distribute the data directly. We provide a series of script that could reproduce our HumanML3D dataset from AMASS dataset. +For KIT-ML dataset, you could directly download [[Here]](https://drive.google.com/drive/folders/1D3bf2G2o4Hv-Ale26YW18r1Wrh7oIAwK?usp=sharing). Due to the distribution policy of AMASS dataset, we are not allowed to distribute the data directly. We provide a series of scripts that could reproduce our HumanML3D dataset from AMASS dataset. You need to clone this repository and install the virtual environment. @@ -102,7 +103,7 @@ After all, the data under folder "./HumanML3D" is what you finally need. ``` HumanML3D data follows the SMPL skeleton structure with 22 joints. KIT-ML has 21 skeletal joints. Refer to paraUtils for detailed kinematic chains. -The file named in "MXXXXXX.\*" (e.g., 'M000000.npy') is mirrored from file with correspinding name "XXXXXX.\*" (e.g., '000000.npy'). Text files and motion files follow the same naming protocols, meaning texts in "./texts/XXXXXX.txt"(e.g., '000000.txt') exactly describe the human motions in "./new_joints(or new_joint_vecs)/XXXXXX.npy" (e.g., '000000.npy') +The file named in "MXXXXXX.\*" (e.g., 'M000000.npy') is mirrored from file with corresponding name "XXXXXX.\*" (e.g., '000000.npy'). Text files and motion files follow the same naming protocols, meaning texts in "./texts/XXXXXX.txt"(e.g., '000000.txt') exactly describe the human motions in "./new_joints(or new_joint_vecs)/XXXXXX.npy" (e.g., '000000.npy') Each text file looks like the following: ```sh @@ -111,11 +112,11 @@ the standing person kicks with their left foot before going back to their origin a man kicks with something or someone with his left leg.#a/DET man/NOUN kick/VERB with/ADP something/PRON or/CCONJ someone/PRON with/ADP his/DET left/ADJ leg/NOUN#0.0#0.0 he is flying kick with his left leg#he/PRON is/AUX fly/VERB kick/NOUN with/ADP his/DET left/ADJ leg/NOUN#0.0#0.0 ``` -with each line a distint textual annotation, composed of four parts: *original description (lower case)*, *processed sentence*, *start time(s)*, *end time(s)*, that are seperated by *#*. +with each line a distinct textual annotation, composed of four parts: *original description (lower case)*, *processed sentence*, *start time(s)*, *end time(s)*, that are separated by *#*. Since some motions are too complicated to be described, we allow the annotators to describe a sub-part of a given motion if required. In these cases, *start time(s)* and *end time(s)* denotes the motion segments that are annotated. Nonetheless, we observe these only occupy a small proportion of HumanML3D. *start time(s)* and *end time(s)* are set to 0 by default, which means the text is captioning the entire sequence of corresponding motion. -If you are not able to install ffmpeg, you could animate videos in '.gif' instead of '.mp4'. However, generating GIFs usually takes longer time and memory occupation. +If you are not able to install ffmpeg, you could animate videos in '.gif' instead of '.mp4'. However, generating GIFs usually takes longer time and uses more memory. ## Citation From 242997676a0db770184f99708cb9ca9a0d83adc1 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 18 Sep 2024 12:01:38 +0800 Subject: [PATCH 04/23] Added .mp4 to .gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 58016b4..130b8b7 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ HumanML3D/texts/*.txt *.gz *.tar.gz *.tgz + +# animations +*.mp4 From 29abc73e28dffeaceb64d0362520372739bc7601 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 6 Nov 2024 10:18:23 +0800 Subject: [PATCH 05/23] Cleaned output/metadata of motion_rep notebook. --- motion_representation.ipynb | 126 ++++-------------------------------- 1 file changed, 11 insertions(+), 115 deletions(-) diff --git a/motion_representation.ipynb b/motion_representation.ipynb index 23d94ce..de1e1a8 100644 --- a/motion_representation.ipynb +++ b/motion_representation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -315,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -326,91 +326,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 1%|▏ | 417/29232 [00:26<24:26, 19.65it/s] " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "011059.npy\n", - "cannot reshape array of size 0 into shape (0,newaxis)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 5%|▍ | 1400/29232 [01:16<21:22, 21.69it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "M009707.npy\n", - "cannot reshape array of size 0 into shape (0,newaxis)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 59%|█████▉ | 17203/29232 [14:08<08:35, 23.35it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "009707.npy\n", - "cannot reshape array of size 0 into shape (0,newaxis)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 88%|████████▊ | 25842/29232 [21:14<02:18, 24.48it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "M011059.npy\n", - "cannot reshape array of size 0 into shape (0,newaxis)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 29232/29232 [24:06<00:00, 20.21it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total clips: 29232, Frames: 4117392, Duration: 3431.160000m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "'''\n", "For HumanML3D Dataset\n", @@ -476,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -486,40 +404,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "abs(reference1 - reference1_1).sum()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "abs(reference2 - reference2_1).sum()" ] From 33fccc2f8c5a84aeaa39076a5ceb2f7856a4c9fd Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 6 Nov 2024 10:23:01 +0800 Subject: [PATCH 06/23] Removed unused last cell. --- motion_representation.ipynb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/motion_representation.ipynb b/motion_representation.ipynb index de1e1a8..4e7bf45 100644 --- a/motion_representation.ipynb +++ b/motion_representation.ipynb @@ -419,13 +419,6 @@ "source": [ "abs(reference2 - reference2_1).sum()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From aa16a3b7918ec2e7946c5d3e62cb0f2d0a083512 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 6 Nov 2024 10:24:48 +0800 Subject: [PATCH 07/23] Added abs position version of motion_rep code - Updated actual motion_representation notebook. - Saved original version as rel_motion_representation.ipynb --- motion_representation.ipynb | 228 +++++++++++++++- rel_motion_representation.ipynb | 445 ++++++++++++++++++++++++++++++++ 2 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 rel_motion_representation.ipynb diff --git a/motion_representation.ipynb b/motion_representation.ipynb index 4e7bf45..e98a856 100644 --- a/motion_representation.ipynb +++ b/motion_representation.ipynb @@ -244,7 +244,203 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_file_abs_root(positions, feet_thre):\n", + " '''\n", + " root information (first 3 values) is absolute instead of relative.\n", + " '''\n", + " # (seq_len, joints_num, 3)\n", + " # '''Down Sample'''\n", + " # positions = positions[::ds_num]\n", + "\n", + " '''Uniform Skeleton'''\n", + " positions = uniform_skeleton(positions, tgt_offsets)\n", + "\n", + " '''Put on Floor'''\n", + " floor_height = positions.min(axis=0).min(axis=0)[1]\n", + " positions[:, :, 1] -= floor_height\n", + " # print(floor_height)\n", + "\n", + " # plot_3d_motion(\"./positions_1.mp4\", kinematic_chain, positions, 'title', fps=20)\n", + "\n", + " '''XZ at origin'''\n", + " root_pos_init = positions[0]\n", + " root_pose_init_xz = root_pos_init[0] * np.array([1, 0, 1])\n", + " positions = positions - root_pose_init_xz\n", + "\n", + " # '''Move the first pose to origin '''\n", + " # root_pos_init = positions[0]\n", + " # positions = positions - root_pos_init[0]\n", + "\n", + " '''All initially face Z+'''\n", + " r_hip, l_hip, sdr_r, sdr_l = face_joint_indx\n", + " across1 = root_pos_init[r_hip] - root_pos_init[l_hip]\n", + " across2 = root_pos_init[sdr_r] - root_pos_init[sdr_l]\n", + " across = across1 + across2\n", + " across = across / np.sqrt((across ** 2).sum(axis=-1))[..., np.newaxis]\n", + "\n", + " # forward (3,), rotate around y-axis\n", + " forward_init = np.cross(np.array([[0, 1, 0]]), across, axis=-1)\n", + " # forward (3,)\n", + " forward_init = forward_init / np.sqrt((forward_init ** 2).sum(axis=-1))[..., np.newaxis]\n", + "\n", + " # print(forward_init)\n", + "\n", + " target = np.array([[0, 0, 1]])\n", + " root_quat_init = qbetween_np(forward_init, target)\n", + " root_quat_init = np.ones(positions.shape[:-1] + (4,)) * root_quat_init\n", + "\n", + " positions_b = positions.copy()\n", + "\n", + " positions = qrot_np(root_quat_init, positions)\n", + "\n", + " # plot_3d_motion(\"./positions_2.mp4\", kinematic_chain, positions, 'title', fps=20)\n", + "\n", + " '''New ground truth positions'''\n", + " global_positions = positions.copy()\n", + "\n", + "# plt.plot(positions_b[:, 0, 0], positions_b[:, 0, 2], marker='*')\n", + "# plt.plot(positions[:, 0, 0], positions[:, 0, 2], marker='o', color='r')\n", + "# plt.xlabel('x')\n", + "# plt.ylabel('z')\n", + "# plt.axis('equal')\n", + "# plt.show()\n", + "\n", + " \"\"\" Get Foot Contacts \"\"\"\n", + "\n", + " def foot_detect(positions, thres):\n", + " velfactor, heightfactor = np.array([thres, thres]), np.array([3.0, 2.0])\n", + "\n", + " feet_l_x = (positions[1:, fid_l, 0] - positions[:-1, fid_l, 0]) ** 2\n", + " feet_l_y = (positions[1:, fid_l, 1] - positions[:-1, fid_l, 1]) ** 2\n", + " feet_l_z = (positions[1:, fid_l, 2] - positions[:-1, fid_l, 2]) ** 2\n", + " # feet_l_h = positions[:-1,fid_l,1]\n", + " # feet_l = (((feet_l_x + feet_l_y + feet_l_z) < velfactor) & (feet_l_h < heightfactor)).astype(np.float)\n", + " feet_l = ((feet_l_x + feet_l_y + feet_l_z) < velfactor).astype(np.float32)\n", + "\n", + " feet_r_x = (positions[1:, fid_r, 0] - positions[:-1, fid_r, 0]) ** 2\n", + " feet_r_y = (positions[1:, fid_r, 1] - positions[:-1, fid_r, 1]) ** 2\n", + " feet_r_z = (positions[1:, fid_r, 2] - positions[:-1, fid_r, 2]) ** 2\n", + " # feet_r_h = positions[:-1,fid_r,1]\n", + " # feet_r = (((feet_r_x + feet_r_y + feet_r_z) < velfactor) & (feet_r_h < heightfactor)).astype(np.float)\n", + " feet_r = (((feet_r_x + feet_r_y + feet_r_z) < velfactor)).astype(np.float32)\n", + " return feet_l, feet_r\n", + " #\n", + " feet_l, feet_r = foot_detect(positions, feet_thre)\n", + " # feet_l, feet_r = foot_detect(positions, 0.002)\n", + "\n", + " '''Quaternion and Cartesian representation'''\n", + " r_rot = None\n", + "\n", + " def get_rifke(positions):\n", + " '''Local pose'''\n", + " positions[..., 0] -= positions[:, 0:1, 0]\n", + " positions[..., 2] -= positions[:, 0:1, 2]\n", + " '''All pose face Z+'''\n", + " positions = qrot_np(np.repeat(r_rot[:, None], positions.shape[1], axis=1), positions)\n", + " return positions\n", + "\n", + " def get_quaternion(positions):\n", + " skel = Skeleton(n_raw_offsets, kinematic_chain, \"cpu\")\n", + " # (seq_len, joints_num, 4)\n", + " quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=False)\n", + "\n", + " '''Fix Quaternion Discontinuity'''\n", + " quat_params = qfix(quat_params)\n", + " # (seq_len, 4)\n", + " r_rot = quat_params[:, 0].copy()\n", + " # print(r_rot[0])\n", + " '''Root Linear Velocity'''\n", + " # (seq_len - 1, 3)\n", + " velocity = (positions[1:, 0] - positions[:-1, 0]).copy()\n", + " # print(r_rot.shape, velocity.shape)\n", + " velocity = qrot_np(r_rot[1:], velocity)\n", + " '''Root Angular Velocity'''\n", + " # (seq_len - 1, 4)\n", + " r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1]))\n", + " quat_params[1:, 0] = r_velocity\n", + " # (seq_len, joints_num, 4)\n", + " return quat_params, r_velocity, velocity, r_rot\n", + "\n", + " def get_cont6d_params(positions):\n", + " skel = Skeleton(n_raw_offsets, kinematic_chain, \"cpu\")\n", + " # (seq_len, joints_num, 4)\n", + " quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=True)\n", + "\n", + " '''Quaternion to continuous 6D'''\n", + " cont_6d_params = quaternion_to_cont6d_np(quat_params)\n", + " # (seq_len, 4)\n", + " r_rot = quat_params[:, 0].copy()\n", + " # print(r_rot[0])\n", + " '''Root Linear Velocity'''\n", + " # (seq_len - 1, 3)\n", + " velocity = (positions[1:, 0] - positions[:-1, 0]).copy()\n", + " # print(r_rot.shape, velocity.shape)\n", + " velocity = qrot_np(r_rot[1:], velocity)\n", + " '''Root Angular Velocity'''\n", + " # (seq_len - 1, 4)\n", + " r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1]))\n", + " # (seq_len, joints_num, 4)\n", + " return cont_6d_params, r_velocity, velocity, r_rot\n", + "\n", + " cont_6d_params, r_velocity, velocity, r_rot = get_cont6d_params(positions)\n", + " positions = get_rifke(positions)\n", + "\n", + "# trejec = np.cumsum(np.concatenate([np.array([[0, 0, 0]]), velocity], axis=0), axis=0)\n", + "# # r_rotations, r_pos = recover_ric_glo_np(r_velocity, velocity[:, [0, 2]])\n", + "\n", + "# # plt.plot(positions_b[:, 0, 0], positions_b[:, 0, 2], marker='*')\n", + "# plt.plot(positions[:, 0, 0], positions[:, 0, 2], marker='*')\n", + "# # plt.plot(ground_positions[:, 0, 0], ground_positions[:, 0, 2], marker='o', color='r')\n", + "# plt.plot(trejec[:, 0], trejec[:, 2], marker='^', color='g')\n", + "# # plt.plot(r_pos[:, 0], r_pos[:, 2], marker='s', color='y')\n", + "# plt.xlabel('x')\n", + "# plt.ylabel('z')\n", + "# plt.axis('equal')\n", + "# plt.show()\n", + "# import pdb; pdb.set_trace()\n", + "\n", + " '''Root height'''\n", + " root_y = positions[:, 0, 1:2]\n", + "\n", + " '''Root rotation and linear velocity'''\n", + " # (seq_len-1, 1) rotation velocity along y-axis\n", + " # (seq_len-1, 2) linear velovity on xz plane\n", + " r_velocity = np.arcsin(r_velocity[:, 2:3])\n", + " l_velocity = velocity[:, [0, 2]]\n", + " # print(r_velocity.shape, l_velocity.shape, root_y.shape)\n", + " root_data = np.concatenate([r_velocity, l_velocity, root_y[:-1]], axis=-1)\n", + "\n", + " '''Get Joint Rotation Representation'''\n", + " # (seq_len, (joints_num-1) *6) quaternion for skeleton joints\n", + " rot_data = cont_6d_params[:, 1:].reshape(len(cont_6d_params), -1)\n", + "\n", + " '''Get Joint Rotation Invariant Position Represention'''\n", + " # (seq_len, (joints_num-1)*3) local joint position\n", + " ric_data = positions[:, 1:].reshape(len(positions), -1)\n", + "\n", + " '''Get Joint Velocity Representation'''\n", + " # (seq_len-1, joints_num*3)\n", + " local_vel = qrot_np(np.repeat(r_rot[:-1, None], global_positions.shape[1], axis=1),\n", + " global_positions[1:] - global_positions[:-1])\n", + " local_vel = local_vel.reshape(len(local_vel), -1)\n", + "\n", + " data = root_data\n", + " data = np.concatenate([data, ric_data[:-1]], axis=-1)\n", + " data = np.concatenate([data, rot_data[:-1]], axis=-1)\n", + " # print(data.shape, local_vel.shape)\n", + " data = np.concatenate([data, local_vel], axis=-1)\n", + " data = np.concatenate([data, feet_l, feet_r], axis=-1)\n", + "\n", + " return data, global_positions, positions, l_velocity\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -256,7 +452,7 @@ "# rot_data (B, seq_len, (joint_num - 1)*6)\n", "# local_velocity (B, seq_len, joint_num*3)\n", "# foot contact (B, seq_len, 4)\n", - "def recover_root_rot_pos(data):\n", + "def recover_root_rot_pos(data, return_rot_ang=False):\n", " rot_vel = data[..., 0]\n", " r_rot_ang = torch.zeros_like(rot_vel).to(data.device)\n", " '''Get Y-axis rotation from rotation velocity'''\n", @@ -275,6 +471,9 @@ " r_pos = torch.cumsum(r_pos, dim=-2)\n", "\n", " r_pos[..., 1] = data[..., 3]\n", + "\n", + " if return_rot_ang:\n", + " return r_rot_quat, r_pos, r_rot_ang\n", " return r_rot_quat, r_pos\n", "\n", "\n", @@ -370,16 +569,37 @@ " for source_file in tqdm(source_list):\n", " source_data = np.load(os.path.join(data_dir, source_file))[:, :joints_num]\n", " try:\n", - " data, ground_positions, positions, l_velocity = process_file(source_data, 0.002)\n", + "# data, ground_positions, positions, l_velocity = process_file(source_data, 0.002)\n", + " ### compute absoluate root information instead of relative, ignore rec_ric_data\n", + " data, ground_positions, positions, l_velocity = process_file_abs_root(source_data, 0.002)\n", " rec_ric_data = recover_from_ric(torch.from_numpy(data).unsqueeze(0).float(), joints_num)\n", + " r_rot_quat, r_pos, rot_ang = recover_root_rot_pos(torch.from_numpy(data), return_rot_ang=True)\n", + " new_data = data.copy()\n", + " new_data[:, 0] = rot_ang\n", + " new_data[:, [1, 2]] = r_pos[:, [0,2]]\n", + " ###\n", " np.save(pjoin(save_dir1, source_file), rec_ric_data.squeeze().numpy())\n", - " np.save(pjoin(save_dir2, source_file), data)\n", + "# np.save(pjoin(save_dir2, source_file), data)\n", + " np.save(pjoin(save_dir2, source_file), new_data)\n", " frame_num += data.shape[0]\n", + " \n", + " # plt.plot(ground_positions[:, 0, 0], ground_positions[:, 0, 2], marker='*')\n", + " # plt.plot(new_data[:, 1], new_data[:, 2], marker='o', color='r')\n", + "# plt.plot(r_pos[:, 0], r_pos[:, 2], marker='o', color='r')\n", + "# plt.xlabel('x')\n", + "# plt.ylabel('z')\n", + "# plt.axis('equal')\n", + "# plt.show()\n", + " # ground_positions[:, 0, [0,2]]\n", + " # new_data[:, [1,2]]\n", + "# import pdb; pdb.set_trace()\n", + " \n", " except Exception as e:\n", " print(source_file)\n", " print(e)\n", "# print(source_file)\n", "# break\n", + "# import pdb; pdb.set_trace()\n", "\n", " print('Total clips: %d, Frames: %d, Duration: %fm' %\n", " (len(source_list), frame_num, frame_num / 20 / 60))" diff --git a/rel_motion_representation.ipynb b/rel_motion_representation.ipynb new file mode 100644 index 0000000..c31974f --- /dev/null +++ b/rel_motion_representation.ipynb @@ -0,0 +1,445 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from os.path import join as pjoin\n", + "\n", + "from common.skeleton import Skeleton\n", + "import numpy as np\n", + "import os\n", + "from common.quaternion import *\n", + "from paramUtil import *\n", + "\n", + "import torch\n", + "from tqdm import tqdm\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def uniform_skeleton(positions, target_offset):\n", + " src_skel = Skeleton(n_raw_offsets, kinematic_chain, 'cpu')\n", + " src_offset = src_skel.get_offsets_joints(torch.from_numpy(positions[0]))\n", + " src_offset = src_offset.numpy()\n", + " tgt_offset = target_offset.numpy()\n", + " # print(src_offset)\n", + " # print(tgt_offset)\n", + " '''Calculate Scale Ratio as the ratio of legs'''\n", + " src_leg_len = np.abs(src_offset[l_idx1]).max() + np.abs(src_offset[l_idx2]).max()\n", + " tgt_leg_len = np.abs(tgt_offset[l_idx1]).max() + np.abs(tgt_offset[l_idx2]).max()\n", + "\n", + " scale_rt = tgt_leg_len / src_leg_len\n", + " # print(scale_rt)\n", + " src_root_pos = positions[:, 0]\n", + " tgt_root_pos = src_root_pos * scale_rt\n", + "\n", + " '''Inverse Kinematics'''\n", + " quat_params = src_skel.inverse_kinematics_np(positions, face_joint_indx)\n", + " # print(quat_params.shape)\n", + "\n", + " '''Forward Kinematics'''\n", + " src_skel.set_offset(target_offset)\n", + " new_joints = src_skel.forward_kinematics_np(quat_params, tgt_root_pos)\n", + " return new_joints\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_file(positions, feet_thre):\n", + " # (seq_len, joints_num, 3)\n", + " # '''Down Sample'''\n", + " # positions = positions[::ds_num]\n", + "\n", + " '''Uniform Skeleton'''\n", + " positions = uniform_skeleton(positions, tgt_offsets)\n", + "\n", + " '''Put on Floor'''\n", + " floor_height = positions.min(axis=0).min(axis=0)[1]\n", + " positions[:, :, 1] -= floor_height\n", + " # print(floor_height)\n", + "\n", + " # plot_3d_motion(\"./positions_1.mp4\", kinematic_chain, positions, 'title', fps=20)\n", + "\n", + " '''XZ at origin'''\n", + " root_pos_init = positions[0]\n", + " root_pose_init_xz = root_pos_init[0] * np.array([1, 0, 1])\n", + " positions = positions - root_pose_init_xz\n", + "\n", + " # '''Move the first pose to origin '''\n", + " # root_pos_init = positions[0]\n", + " # positions = positions - root_pos_init[0]\n", + "\n", + " '''All initially face Z+'''\n", + " r_hip, l_hip, sdr_r, sdr_l = face_joint_indx\n", + " across1 = root_pos_init[r_hip] - root_pos_init[l_hip]\n", + " across2 = root_pos_init[sdr_r] - root_pos_init[sdr_l]\n", + " across = across1 + across2\n", + " across = across / np.sqrt((across ** 2).sum(axis=-1))[..., np.newaxis]\n", + "\n", + " # forward (3,), rotate around y-axis\n", + " forward_init = np.cross(np.array([[0, 1, 0]]), across, axis=-1)\n", + " # forward (3,)\n", + " forward_init = forward_init / np.sqrt((forward_init ** 2).sum(axis=-1))[..., np.newaxis]\n", + "\n", + " # print(forward_init)\n", + "\n", + " target = np.array([[0, 0, 1]])\n", + " root_quat_init = qbetween_np(forward_init, target)\n", + " root_quat_init = np.ones(positions.shape[:-1] + (4,)) * root_quat_init\n", + "\n", + " positions_b = positions.copy()\n", + "\n", + " positions = qrot_np(root_quat_init, positions)\n", + "\n", + " # plot_3d_motion(\"./positions_2.mp4\", kinematic_chain, positions, 'title', fps=20)\n", + "\n", + " '''New ground truth positions'''\n", + " global_positions = positions.copy()\n", + "\n", + " # plt.plot(positions_b[:, 0, 0], positions_b[:, 0, 2], marker='*')\n", + " # plt.plot(positions[:, 0, 0], positions[:, 0, 2], marker='o', color='r')\n", + " # plt.xlabel('x')\n", + " # plt.ylabel('z')\n", + " # plt.axis('equal')\n", + " # plt.show()\n", + "\n", + " \"\"\" Get Foot Contacts \"\"\"\n", + "\n", + " def foot_detect(positions, thres):\n", + " velfactor, heightfactor = np.array([thres, thres]), np.array([3.0, 2.0])\n", + "\n", + " feet_l_x = (positions[1:, fid_l, 0] - positions[:-1, fid_l, 0]) ** 2\n", + " feet_l_y = (positions[1:, fid_l, 1] - positions[:-1, fid_l, 1]) ** 2\n", + " feet_l_z = (positions[1:, fid_l, 2] - positions[:-1, fid_l, 2]) ** 2\n", + " # feet_l_h = positions[:-1,fid_l,1]\n", + " # feet_l = (((feet_l_x + feet_l_y + feet_l_z) < velfactor) & (feet_l_h < heightfactor)).astype(np.float)\n", + " feet_l = ((feet_l_x + feet_l_y + feet_l_z) < velfactor).astype(np.float32)\n", + "\n", + " feet_r_x = (positions[1:, fid_r, 0] - positions[:-1, fid_r, 0]) ** 2\n", + " feet_r_y = (positions[1:, fid_r, 1] - positions[:-1, fid_r, 1]) ** 2\n", + " feet_r_z = (positions[1:, fid_r, 2] - positions[:-1, fid_r, 2]) ** 2\n", + " # feet_r_h = positions[:-1,fid_r,1]\n", + " # feet_r = (((feet_r_x + feet_r_y + feet_r_z) < velfactor) & (feet_r_h < heightfactor)).astype(np.float)\n", + " feet_r = (((feet_r_x + feet_r_y + feet_r_z) < velfactor)).astype(np.float32)\n", + " return feet_l, feet_r\n", + " #\n", + " feet_l, feet_r = foot_detect(positions, feet_thre)\n", + " # feet_l, feet_r = foot_detect(positions, 0.002)\n", + "\n", + " '''Quaternion and Cartesian representation'''\n", + " r_rot = None\n", + "\n", + " def get_rifke(positions):\n", + " '''Local pose'''\n", + " positions[..., 0] -= positions[:, 0:1, 0]\n", + " positions[..., 2] -= positions[:, 0:1, 2]\n", + " '''All pose face Z+'''\n", + " positions = qrot_np(np.repeat(r_rot[:, None], positions.shape[1], axis=1), positions)\n", + " return positions\n", + "\n", + " def get_quaternion(positions):\n", + " skel = Skeleton(n_raw_offsets, kinematic_chain, \"cpu\")\n", + " # (seq_len, joints_num, 4)\n", + " quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=False)\n", + "\n", + " '''Fix Quaternion Discontinuity'''\n", + " quat_params = qfix(quat_params)\n", + " # (seq_len, 4)\n", + " r_rot = quat_params[:, 0].copy()\n", + " # print(r_rot[0])\n", + " '''Root Linear Velocity'''\n", + " # (seq_len - 1, 3)\n", + " velocity = (positions[1:, 0] - positions[:-1, 0]).copy()\n", + " # print(r_rot.shape, velocity.shape)\n", + " velocity = qrot_np(r_rot[1:], velocity)\n", + " '''Root Angular Velocity'''\n", + " # (seq_len - 1, 4)\n", + " r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1]))\n", + " quat_params[1:, 0] = r_velocity\n", + " # (seq_len, joints_num, 4)\n", + " return quat_params, r_velocity, velocity, r_rot\n", + "\n", + " def get_cont6d_params(positions):\n", + " skel = Skeleton(n_raw_offsets, kinematic_chain, \"cpu\")\n", + " # (seq_len, joints_num, 4)\n", + " quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=True)\n", + "\n", + " '''Quaternion to continuous 6D'''\n", + " cont_6d_params = quaternion_to_cont6d_np(quat_params)\n", + " # (seq_len, 4)\n", + " r_rot = quat_params[:, 0].copy()\n", + " # print(r_rot[0])\n", + " '''Root Linear Velocity'''\n", + " # (seq_len - 1, 3)\n", + " velocity = (positions[1:, 0] - positions[:-1, 0]).copy()\n", + " # print(r_rot.shape, velocity.shape)\n", + " velocity = qrot_np(r_rot[1:], velocity)\n", + " '''Root Angular Velocity'''\n", + " # (seq_len - 1, 4)\n", + " r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1]))\n", + " # (seq_len, joints_num, 4)\n", + " return cont_6d_params, r_velocity, velocity, r_rot\n", + "\n", + " cont_6d_params, r_velocity, velocity, r_rot = get_cont6d_params(positions)\n", + " positions = get_rifke(positions)\n", + "\n", + " # trejec = np.cumsum(np.concatenate([np.array([[0, 0, 0]]), velocity], axis=0), axis=0)\n", + " # r_rotations, r_pos = recover_ric_glo_np(r_velocity, velocity[:, [0, 2]])\n", + "\n", + " # plt.plot(positions_b[:, 0, 0], positions_b[:, 0, 2], marker='*')\n", + " # plt.plot(ground_positions[:, 0, 0], ground_positions[:, 0, 2], marker='o', color='r')\n", + " # plt.plot(trejec[:, 0], trejec[:, 2], marker='^', color='g')\n", + " # plt.plot(r_pos[:, 0], r_pos[:, 2], marker='s', color='y')\n", + " # plt.xlabel('x')\n", + " # plt.ylabel('z')\n", + " # plt.axis('equal')\n", + " # plt.show()\n", + "\n", + " '''Root height'''\n", + " root_y = positions[:, 0, 1:2]\n", + "\n", + " '''Root rotation and linear velocity'''\n", + " # (seq_len-1, 1) rotation velocity along y-axis\n", + " # (seq_len-1, 2) linear velovity on xz plane\n", + " r_velocity = np.arcsin(r_velocity[:, 2:3])\n", + " l_velocity = velocity[:, [0, 2]]\n", + " # print(r_velocity.shape, l_velocity.shape, root_y.shape)\n", + " root_data = np.concatenate([r_velocity, l_velocity, root_y[:-1]], axis=-1)\n", + "\n", + " '''Get Joint Rotation Representation'''\n", + " # (seq_len, (joints_num-1) *6) quaternion for skeleton joints\n", + " rot_data = cont_6d_params[:, 1:].reshape(len(cont_6d_params), -1)\n", + "\n", + " '''Get Joint Rotation Invariant Position Represention'''\n", + " # (seq_len, (joints_num-1)*3) local joint position\n", + " ric_data = positions[:, 1:].reshape(len(positions), -1)\n", + "\n", + " '''Get Joint Velocity Representation'''\n", + " # (seq_len-1, joints_num*3)\n", + " local_vel = qrot_np(np.repeat(r_rot[:-1, None], global_positions.shape[1], axis=1),\n", + " global_positions[1:] - global_positions[:-1])\n", + " local_vel = local_vel.reshape(len(local_vel), -1)\n", + "\n", + " data = root_data\n", + " data = np.concatenate([data, ric_data[:-1]], axis=-1)\n", + " data = np.concatenate([data, rot_data[:-1]], axis=-1)\n", + " # print(data.shape, local_vel.shape)\n", + " data = np.concatenate([data, local_vel], axis=-1)\n", + " data = np.concatenate([data, feet_l, feet_r], axis=-1)\n", + "\n", + " return data, global_positions, positions, l_velocity\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Recover global angle and positions for rotation data\n", + "# root_rot_velocity (B, seq_len, 1)\n", + "# root_linear_velocity (B, seq_len, 2)\n", + "# root_y (B, seq_len, 1)\n", + "# ric_data (B, seq_len, (joint_num - 1)*3)\n", + "# rot_data (B, seq_len, (joint_num - 1)*6)\n", + "# local_velocity (B, seq_len, joint_num*3)\n", + "# foot contact (B, seq_len, 4)\n", + "def recover_root_rot_pos(data):\n", + " rot_vel = data[..., 0]\n", + " r_rot_ang = torch.zeros_like(rot_vel).to(data.device)\n", + " '''Get Y-axis rotation from rotation velocity'''\n", + " r_rot_ang[..., 1:] = rot_vel[..., :-1]\n", + " r_rot_ang = torch.cumsum(r_rot_ang, dim=-1)\n", + "\n", + " r_rot_quat = torch.zeros(data.shape[:-1] + (4,)).to(data.device)\n", + " r_rot_quat[..., 0] = torch.cos(r_rot_ang)\n", + " r_rot_quat[..., 2] = torch.sin(r_rot_ang)\n", + "\n", + " r_pos = torch.zeros(data.shape[:-1] + (3,)).to(data.device)\n", + " r_pos[..., 1:, [0, 2]] = data[..., :-1, 1:3]\n", + " '''Add Y-axis rotation to root position'''\n", + " r_pos = qrot(qinv(r_rot_quat), r_pos)\n", + "\n", + " r_pos = torch.cumsum(r_pos, dim=-2)\n", + "\n", + " r_pos[..., 1] = data[..., 3]\n", + " return r_rot_quat, r_pos\n", + "\n", + "\n", + "def recover_from_rot(data, joints_num, skeleton):\n", + " r_rot_quat, r_pos = recover_root_rot_pos(data)\n", + "\n", + " r_rot_cont6d = quaternion_to_cont6d(r_rot_quat)\n", + "\n", + " start_indx = 1 + 2 + 1 + (joints_num - 1) * 3\n", + " end_indx = start_indx + (joints_num - 1) * 6\n", + " cont6d_params = data[..., start_indx:end_indx]\n", + " # print(r_rot_cont6d.shape, cont6d_params.shape, r_pos.shape)\n", + " cont6d_params = torch.cat([r_rot_cont6d, cont6d_params], dim=-1)\n", + " cont6d_params = cont6d_params.view(-1, joints_num, 6)\n", + "\n", + " positions = skeleton.forward_kinematics_cont6d(cont6d_params, r_pos)\n", + "\n", + " return positions\n", + "\n", + "\n", + "def recover_from_ric(data, joints_num):\n", + " r_rot_quat, r_pos = recover_root_rot_pos(data)\n", + " positions = data[..., 4:(joints_num - 1) * 3 + 4]\n", + " positions = positions.view(positions.shape[:-1] + (-1, 3))\n", + "\n", + " '''Add Y-axis rotation to local joints'''\n", + " positions = qrot(qinv(r_rot_quat[..., None, :]).expand(positions.shape[:-1] + (4,)), positions)\n", + "\n", + " '''Add root XZ to joints'''\n", + " positions[..., 0] += r_pos[..., 0:1]\n", + " positions[..., 2] += r_pos[..., 2:3]\n", + "\n", + " '''Concate root and joints'''\n", + " positions = torch.cat([r_pos.unsqueeze(-2), positions], dim=-2)\n", + "\n", + " return positions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The given data is used to double check if you are on the right track.\n", + "reference1 = np.load('./HumanML3D/new_joints/012314.npy')\n", + "reference2 = np.load('./HumanML3D/new_joint_vecs/012314.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "'''\n", + "For HumanML3D Dataset\n", + "'''\n", + "\n", + "if __name__ == \"__main__\":\n", + " example_id = \"000021\"\n", + " # Lower legs\n", + " l_idx1, l_idx2 = 5, 8\n", + " # Right/Left foot\n", + " fid_r, fid_l = [8, 11], [7, 10]\n", + " # Face direction, r_hip, l_hip, sdr_r, sdr_l\n", + " face_joint_indx = [2, 1, 17, 16]\n", + " # l_hip, r_hip\n", + " r_hip, l_hip = 2, 1\n", + " joints_num = 22\n", + " # ds_num = 8\n", + " data_dir = './joints/'\n", + " save_dir1 = './HumanML3D/new_joints/'\n", + " save_dir2 = './HumanML3D/new_joint_vecs/'\n", + " \n", + " os.makedirs(save_dir1, exist_ok=True)\n", + " os.makedirs(save_dir2, exist_ok=True)\n", + "\n", + " n_raw_offsets = torch.from_numpy(t2m_raw_offsets)\n", + " kinematic_chain = t2m_kinematic_chain\n", + "\n", + " # Get offsets of target skeleton\n", + " example_data = np.load(os.path.join(data_dir, example_id + '.npy'))\n", + " example_data = example_data.reshape(len(example_data), -1, 3)\n", + " example_data = torch.from_numpy(example_data)\n", + " tgt_skel = Skeleton(n_raw_offsets, kinematic_chain, 'cpu')\n", + " # (joints_num, 3)\n", + " tgt_offsets = tgt_skel.get_offsets_joints(example_data[0])\n", + " # print(tgt_offsets)\n", + "\n", + " source_list = os.listdir(data_dir)\n", + " frame_num = 0\n", + " for source_file in tqdm(source_list):\n", + " source_data = np.load(os.path.join(data_dir, source_file))[:, :joints_num]\n", + " try:\n", + " data, ground_positions, positions, l_velocity = process_file(source_data, 0.002)\n", + " rec_ric_data = recover_from_ric(torch.from_numpy(data).unsqueeze(0).float(), joints_num)\n", + " np.save(pjoin(save_dir1, source_file), rec_ric_data.squeeze().numpy())\n", + " np.save(pjoin(save_dir2, source_file), data)\n", + " frame_num += data.shape[0]\n", + " except Exception as e:\n", + " print(source_file)\n", + " print(e)\n", + "# print(source_file)\n", + "# break\n", + "\n", + " print('Total clips: %d, Frames: %d, Duration: %fm' %\n", + " (len(source_list), frame_num, frame_num / 20 / 60))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check if your data is correct. If it's aligned with the given reference, then it is right" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reference1_1 = np.load('./HumanML3D/new_joints/012314.npy')\n", + "reference2_1 = np.load('./HumanML3D/new_joint_vecs/012314.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "abs(reference1 - reference1_1).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "abs(reference2 - reference2_1).sum()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hml3d", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 7bc5f6ec5a1bbe46acde7e5c520b585785a909ed Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 6 Nov 2024 10:31:51 +0800 Subject: [PATCH 08/23] Cleaned output for all base notebooks - updated venv name and python versions - removed unused empty cells at end of each notebook --- animation.ipynb | 35 ++++++--------------- cal_mean_variance.ipynb | 61 +++++++------------------------------ motion_representation.ipynb | 6 ++-- 3 files changed, 24 insertions(+), 78 deletions(-) diff --git a/animation.ipynb b/animation.ipynb index b07d12f..69afb8c 100644 --- a/animation.ipynb +++ b/animation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -112,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -122,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -132,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -157,17 +157,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 10/10 [00:14<00:00, 1.43s/it]\n" - ] - } - ], + "outputs": [], "source": [ "for npy_file in tqdm(npy_files):\n", " data = np.load(pjoin(src_dir, npy_file))\n", @@ -177,20 +169,13 @@ "# You may set the title on your own.\n", " plot_3d_motion(save_path, kinematic_chain, data, title=\"None\", fps=20, radius=4)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:torch_render]", + "display_name": "hml3d", "language": "python", - "name": "conda-env-torch_render-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -202,7 +187,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/cal_mean_variance.ipynb b/cal_mean_variance.ipynb index a9070c5..af96899 100644 --- a/cal_mean_variance.ipynb +++ b/cal_mean_variance.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -63,19 +63,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "007975.npy\n", - "M007975.npy\n", - "(4117224, 263)\n" - ] - } - ], + "outputs": [], "source": [ "if __name__ == '__main__':\n", " data_dir = './HumanML3D/new_joint_vecs/'\n", @@ -94,57 +84,28 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "abs(mean-reference1).sum()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "abs(std-reference2).sum()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:torch_render]", + "display_name": "hml3d", "language": "python", - "name": "conda-env-torch_render-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -156,7 +117,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/motion_representation.ipynb b/motion_representation.ipynb index e98a856..82f5b8d 100644 --- a/motion_representation.ipynb +++ b/motion_representation.ipynb @@ -643,9 +643,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:torch_render]", + "display_name": "hml3d", "language": "python", - "name": "conda-env-torch_render-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -657,7 +657,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.19" } }, "nbformat": 4, From c876c2589b1d6f97d6682d6a737e7c6a6dce92b6 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 6 Nov 2024 10:59:09 +0800 Subject: [PATCH 09/23] Updated mean and variance calculation notebook. - converted to "absolute value" version - really just a path change - saved base version as rel_cal_mean_variance.ipynb --- cal_mean_variance.ipynb | 6 +- rel_cal_mean_variance.ipynb | 132 ++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 rel_cal_mean_variance.ipynb diff --git a/cal_mean_variance.ipynb b/cal_mean_variance.ipynb index af96899..a91e28d 100644 --- a/cal_mean_variance.ipynb +++ b/cal_mean_variance.ipynb @@ -44,8 +44,8 @@ "\n", " assert 8 + (joints_num - 1) * 9 + joints_num * 3 == Std.shape[-1]\n", "\n", - " np.save(pjoin(save_dir, 'Mean.npy'), Mean)\n", - " np.save(pjoin(save_dir, 'Std.npy'), Std)\n", + " np.save(pjoin(save_dir, 'Mean_abs_3d.npy'), Mean)\n", + " np.save(pjoin(save_dir, 'Std_abs_3d.npy'), Std)\n", "\n", " return Mean, Std" ] @@ -68,7 +68,7 @@ "outputs": [], "source": [ "if __name__ == '__main__':\n", - " data_dir = './HumanML3D/new_joint_vecs/'\n", + " data_dir = './HumanML3D/new_joint_vecs_abs_3d/'\n", " save_dir = './HumanML3D/'\n", " mean, std = mean_variance(data_dir, save_dir, 22)\n", "# print(mean)\n", diff --git a/rel_cal_mean_variance.ipynb b/rel_cal_mean_variance.ipynb new file mode 100644 index 0000000..9dbc8f5 --- /dev/null +++ b/rel_cal_mean_variance.ipynb @@ -0,0 +1,132 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import sys\n", + "import os\n", + "from os.path import join as pjoin\n", + "\n", + "\n", + "# root_rot_velocity (B, seq_len, 1)\n", + "# root_linear_velocity (B, seq_len, 2)\n", + "# root_y (B, seq_len, 1)\n", + "# ric_data (B, seq_len, (joint_num - 1)*3)\n", + "# rot_data (B, seq_len, (joint_num - 1)*6)\n", + "# local_velocity (B, seq_len, joint_num*3)\n", + "# foot contact (B, seq_len, 4)\n", + "def mean_variance(data_dir, save_dir, joints_num):\n", + " file_list = os.listdir(data_dir)\n", + " data_list = []\n", + "\n", + " for file in file_list:\n", + " data = np.load(pjoin(data_dir, file))\n", + " if np.isnan(data).any():\n", + " print(file)\n", + " continue\n", + " data_list.append(data)\n", + "\n", + " data = np.concatenate(data_list, axis=0)\n", + " print(data.shape)\n", + " Mean = data.mean(axis=0)\n", + " Std = data.std(axis=0)\n", + " Std[0:1] = Std[0:1].mean() / 1.0\n", + " Std[1:3] = Std[1:3].mean() / 1.0\n", + " Std[3:4] = Std[3:4].mean() / 1.0\n", + " Std[4: 4+(joints_num - 1) * 3] = Std[4: 4+(joints_num - 1) * 3].mean() / 1.0\n", + " Std[4+(joints_num - 1) * 3: 4+(joints_num - 1) * 9] = Std[4+(joints_num - 1) * 3: 4+(joints_num - 1) * 9].mean() / 1.0\n", + " Std[4+(joints_num - 1) * 9: 4+(joints_num - 1) * 9 + joints_num*3] = Std[4+(joints_num - 1) * 9: 4+(joints_num - 1) * 9 + joints_num*3].mean() / 1.0\n", + " Std[4 + (joints_num - 1) * 9 + joints_num * 3: ] = Std[4 + (joints_num - 1) * 9 + joints_num * 3: ].mean() / 1.0\n", + "\n", + " assert 8 + (joints_num - 1) * 9 + joints_num * 3 == Std.shape[-1]\n", + "\n", + " np.save(pjoin(save_dir, 'Mean.npy'), Mean)\n", + " np.save(pjoin(save_dir, 'Std.npy'), Std)\n", + "\n", + " return Mean, Std" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The given data is used to double check if you are on the right track.\n", + "reference1 = np.load('./HumanML3D/Mean.npy')\n", + "reference2 = np.load('./HumanML3D/Std.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == '__main__':\n", + " data_dir = './HumanML3D/new_joint_vecs/'\n", + " save_dir = './HumanML3D/'\n", + " mean, std = mean_variance(data_dir, save_dir, 22)\n", + "# print(mean)\n", + "# print(Std)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check if your data is correct. If it's aligned with the given reference, then it is right" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "abs(mean-reference1).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "abs(std-reference2).sum()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "hml3d", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 0ed383150e78e91c35524e5028829d37c34768d9 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 20 Nov 2024 16:28:57 +0800 Subject: [PATCH 10/23] Added base working version of build_vector.py. --- build_vector.py | 291 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 build_vector.py diff --git a/build_vector.py b/build_vector.py new file mode 100644 index 0000000..66af42d --- /dev/null +++ b/build_vector.py @@ -0,0 +1,291 @@ +from os.path import join as pjoin + +from common.skeleton import Skeleton +import numpy as np +import os +from common.quaternion import * +from paramUtil import * + +import torch +from tqdm import tqdm +import os + +def uniform_skeleton(positions, target_offset): + src_skel = Skeleton(n_raw_offsets, kinematic_chain, 'cpu') + src_offset = src_skel.get_offsets_joints(torch.from_numpy(positions[0])) + src_offset = src_offset.numpy() + tgt_offset = target_offset.numpy() + # print(src_offset) + # print(tgt_offset) + '''Calculate Scale Ratio as the ratio of legs''' + src_leg_len = np.abs(src_offset[l_idx1]).max() + np.abs(src_offset[l_idx2]).max() + tgt_leg_len = np.abs(tgt_offset[l_idx1]).max() + np.abs(tgt_offset[l_idx2]).max() + + scale_rt = tgt_leg_len / src_leg_len + # print(scale_rt) + src_root_pos = positions[:, 0] + tgt_root_pos = src_root_pos * scale_rt + + '''Inverse Kinematics''' + quat_params = src_skel.inverse_kinematics_np(positions, face_joint_indx) + # print(quat_params.shape) + + '''Forward Kinematics''' + src_skel.set_offset(target_offset) + new_joints = src_skel.forward_kinematics_np(quat_params, tgt_root_pos) + return new_joints + + +def process_file_abs_root(positions, feet_thre): + + '''Uniform Skeleton''' + positions = uniform_skeleton(positions, tgt_offsets) + + '''Put on Floor''' + floor_height = positions.min(axis=0).min(axis=0)[1] + positions[:, :, 1] -= floor_height + + '''XZ at origin''' + root_pos_init = positions[0] + root_pose_init_xz = root_pos_init[0] * np.array([1, 0, 1]) + positions = positions - root_pose_init_xz + + '''All initially face Z+''' + r_hip, l_hip, sdr_r, sdr_l = face_joint_indx + across1 = root_pos_init[r_hip] - root_pos_init[l_hip] + across2 = root_pos_init[sdr_r] - root_pos_init[sdr_l] + across = across1 + across2 + across = across / np.sqrt((across ** 2).sum(axis=-1))[..., np.newaxis] + + forward_init = np.cross(np.array([[0, 1, 0]]), across, axis=-1) + forward_init = forward_init / np.sqrt((forward_init ** 2).sum(axis=-1))[..., np.newaxis] + + + target = np.array([[0, 0, 1]]) + root_quat_init = qbetween_np(forward_init, target) + root_quat_init = np.ones(positions.shape[:-1] + (4,)) * root_quat_init + + positions_b = positions.copy() + + positions = qrot_np(root_quat_init, positions) + + + '''New ground truth positions''' + global_positions = positions.copy() + + ''' Get Foot Contacts ''' + + def foot_detect(positions, thres): + velfactor, heightfactor = np.array([thres, thres]), np.array([3.0, 2.0]) + + feet_l_x = (positions[1:, fid_l, 0] - positions[:-1, fid_l, 0]) ** 2 + feet_l_y = (positions[1:, fid_l, 1] - positions[:-1, fid_l, 1]) ** 2 + feet_l_z = (positions[1:, fid_l, 2] - positions[:-1, fid_l, 2]) ** 2 + feet_l = ((feet_l_x + feet_l_y + feet_l_z) < velfactor).astype(np.float32) + + feet_r_x = (positions[1:, fid_r, 0] - positions[:-1, fid_r, 0]) ** 2 + feet_r_y = (positions[1:, fid_r, 1] - positions[:-1, fid_r, 1]) ** 2 + feet_r_z = (positions[1:, fid_r, 2] - positions[:-1, fid_r, 2]) ** 2 + feet_r = (((feet_r_x + feet_r_y + feet_r_z) < velfactor)).astype(np.float32) + return feet_l, feet_r + + feet_l, feet_r = foot_detect(positions, feet_thre) + + '''Quaternion and Cartesian representation''' + r_rot = None + + def get_rifke(positions): + '''Local pose''' + positions[..., 0] -= positions[:, 0:1, 0] + positions[..., 2] -= positions[:, 0:1, 2] + '''All pose face Z+''' + positions = qrot_np(np.repeat(r_rot[:, None], positions.shape[1], axis=1), positions) + return positions + + def get_quaternion(positions): + skel = Skeleton(n_raw_offsets, kinematic_chain, "cpu") + quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=False) + + '''Fix Quaternion Discontinuity''' + quat_params = qfix(quat_params) + r_rot = quat_params[:, 0].copy() + '''Root Linear Velocity''' + velocity = (positions[1:, 0] - positions[:-1, 0]).copy() + velocity = qrot_np(r_rot[1:], velocity) + '''Root Angular Velocity''' + r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1])) + quat_params[1:, 0] = r_velocity + return quat_params, r_velocity, velocity, r_rot + + def get_cont6d_params(positions): + skel = Skeleton(n_raw_offsets, kinematic_chain, "cpu") + quat_params = skel.inverse_kinematics_np(positions, face_joint_indx, smooth_forward=True) + + '''Quaternion to continuous 6D''' + cont_6d_params = quaternion_to_cont6d_np(quat_params) + r_rot = quat_params[:, 0].copy() + '''Root Linear Velocity''' + velocity = (positions[1:, 0] - positions[:-1, 0]).copy() + velocity = qrot_np(r_rot[1:], velocity) + '''Root Angular Velocity''' + r_velocity = qmul_np(r_rot[1:], qinv_np(r_rot[:-1])) + return cont_6d_params, r_velocity, velocity, r_rot + + cont_6d_params, r_velocity, velocity, r_rot = get_cont6d_params(positions) + positions = get_rifke(positions) + + '''Root height''' + root_y = positions[:, 0, 1:2] + + '''Root rotation and linear velocity''' + r_velocity = np.arcsin(r_velocity[:, 2:3]) + l_velocity = velocity[:, [0, 2]] + root_data = np.concatenate([r_velocity, l_velocity, root_y[:-1]], axis=-1) + + '''Get Joint Rotation Representation''' + rot_data = cont_6d_params[:, 1:].reshape(len(cont_6d_params), -1) + + '''Get Joint Rotation Invariant Position Represention''' + ric_data = positions[:, 1:].reshape(len(positions), -1) + + '''Get Joint Velocity Representation''' + local_vel = qrot_np(np.repeat(r_rot[:-1, None], global_positions.shape[1], axis=1), + global_positions[1:] - global_positions[:-1]) + local_vel = local_vel.reshape(len(local_vel), -1) + + data = root_data + data = np.concatenate([data, ric_data[:-1]], axis=-1) + data = np.concatenate([data, rot_data[:-1]], axis=-1) + data = np.concatenate([data, local_vel], axis=-1) + data = np.concatenate([data, feet_l, feet_r], axis=-1) + + return data, global_positions, positions, l_velocity + + +# Recover global angle and positions for rotation data +# root_rot_velocity (B, seq_len, 1) +# root_linear_velocity (B, seq_len, 2) +# root_y (B, seq_len, 1) +# ric_data (B, seq_len, (joint_num - 1)*3) +# rot_data (B, seq_len, (joint_num - 1)*6) +# local_velocity (B, seq_len, joint_num*3) +# foot contact (B, seq_len, 4) +def recover_root_rot_pos(data, return_rot_ang=False): + rot_vel = data[..., 0] + r_rot_ang = torch.zeros_like(rot_vel).to(data.device) + '''Get Y-axis rotation from rotation velocity''' + r_rot_ang[..., 1:] = rot_vel[..., :-1] + r_rot_ang = torch.cumsum(r_rot_ang, dim=-1) + + r_rot_quat = torch.zeros(data.shape[:-1] + (4,)).to(data.device) + r_rot_quat[..., 0] = torch.cos(r_rot_ang) + r_rot_quat[..., 2] = torch.sin(r_rot_ang) + + r_pos = torch.zeros(data.shape[:-1] + (3,)).to(data.device) + r_pos[..., 1:, [0, 2]] = data[..., :-1, 1:3] + '''Add Y-axis rotation to root position''' + r_pos = qrot(qinv(r_rot_quat), r_pos) + + r_pos = torch.cumsum(r_pos, dim=-2) + + r_pos[..., 1] = data[..., 3] + + if return_rot_ang: + return r_rot_quat, r_pos, r_rot_ang + return r_rot_quat, r_pos + + +def recover_from_rot(data, joints_num, skeleton): + r_rot_quat, r_pos = recover_root_rot_pos(data) + + r_rot_cont6d = quaternion_to_cont6d(r_rot_quat) + + start_indx = 1 + 2 + 1 + (joints_num - 1) * 3 + end_indx = start_indx + (joints_num - 1) * 6 + cont6d_params = data[..., start_indx:end_indx] + # print(r_rot_cont6d.shape, cont6d_params.shape, r_pos.shape) + cont6d_params = torch.cat([r_rot_cont6d, cont6d_params], dim=-1) + cont6d_params = cont6d_params.view(-1, joints_num, 6) + + positions = skeleton.forward_kinematics_cont6d(cont6d_params, r_pos) + + return positions + + +def recover_from_ric(data, joints_num): + r_rot_quat, r_pos = recover_root_rot_pos(data) + positions = data[..., 4:(joints_num - 1) * 3 + 4] + positions = positions.view(positions.shape[:-1] + (-1, 3)) + + '''Add Y-axis rotation to local joints''' + positions = qrot(qinv(r_rot_quat[..., None, :]).expand(positions.shape[:-1] + (4,)), positions) + + '''Add root XZ to joints''' + positions[..., 0] += r_pos[..., 0:1] + positions[..., 2] += r_pos[..., 2:3] + + '''Concate root and joints''' + positions = torch.cat([r_pos.unsqueeze(-2), positions], dim=-2) + + return positions + + + +if __name__ == "__main__": + + ## data for existing rig + ## orig. 5, 8; [8, 11], [7, 10]; [2, 1, 17, 16]; 2, 1; 22 (000021) + ## cf. 6, 1; [9, 10], [4, 5]; [6, 1, 23, 18]; 6, 1; 27 (003) + example_id = "000021" + # Lower legs + l_idx1, l_idx2 = 5, 8 + # Right/Left foot + fid_r, fid_l = [8, 11], [7, 10] + # Face direction, r_hip, l_hip, sdr_r, sdr_l + face_joint_indx = [2, 1, 17, 16] + # l_hip, r_hip + r_hip, l_hip = 2, 1 + joints_num = 22 + # ds_num = 8 + data_dir = './joints/' + save_dir1 = './HumanML3D/new_joints/' + save_dir2 = './HumanML3D/new_joint_vecs/' + + os.makedirs(save_dir1, exist_ok=True) + os.makedirs(save_dir2, exist_ok=True) + + n_raw_offsets = torch.from_numpy(t2m_raw_offsets) + kinematic_chain = t2m_kinematic_chain + + # Get offsets of target skeleton + example_data = np.load(os.path.join(data_dir, example_id + '.npy')) + example_data = example_data.reshape(len(example_data), -1, 3) + example_data = torch.from_numpy(example_data) + tgt_skel = Skeleton(n_raw_offsets, kinematic_chain, 'cpu') + # (joints_num, 3) + tgt_offsets = tgt_skel.get_offsets_joints(example_data[0]) + # print(tgt_offsets) + + source_list = os.listdir(data_dir) + frame_num = 0 + for source_file in tqdm(source_list): + source_data = np.load(os.path.join(data_dir, source_file))[:, :joints_num] + try: + ### compute absolute root information instead of relative, ignore rec_ric_data + data, ground_positions, positions, l_velocity = process_file_abs_root(source_data, 0.002) + rec_ric_data = recover_from_ric(torch.from_numpy(data).unsqueeze(0).float(), joints_num) + r_rot_quat, r_pos, rot_ang = recover_root_rot_pos(torch.from_numpy(data), return_rot_ang=True) + new_data = data.copy() + new_data[:, 0] = rot_ang + new_data[:, [1, 2]] = r_pos[:, [0,2]] + + np.save(pjoin(save_dir1, source_file), rec_ric_data.squeeze().numpy()) + np.save(pjoin(save_dir2, source_file), new_data) + frame_num += data.shape[0] + + except Exception as e: + print(source_file) + print(e) + + print('Total clips: %d, Frames: %d, Duration: %fm' % + (len(source_list), frame_num, frame_num / 20 / 60)) From 0cca91561bd9f26a0f98c9ce44bc9e49b6c82580 Mon Sep 17 00:00:00 2001 From: icedwater Date: Wed, 20 Nov 2024 16:45:52 +0800 Subject: [PATCH 11/23] Updated the import structure - used custom_paramUtils and specified import variables - other cleanups --- build_vector.py | 5 ++-- custom_paramUtil.py | 71 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 custom_paramUtil.py diff --git a/build_vector.py b/build_vector.py index 66af42d..692f969 100644 --- a/build_vector.py +++ b/build_vector.py @@ -4,11 +4,10 @@ import numpy as np import os from common.quaternion import * -from paramUtil import * +from custom_paramUtil import t2m_kinematic_chain, t2m_raw_offsets, t2m_tgt_skel_id import torch from tqdm import tqdm -import os def uniform_skeleton(positions, target_offset): src_skel = Skeleton(n_raw_offsets, kinematic_chain, 'cpu') @@ -236,7 +235,7 @@ def recover_from_ric(data, joints_num): ## data for existing rig ## orig. 5, 8; [8, 11], [7, 10]; [2, 1, 17, 16]; 2, 1; 22 (000021) ## cf. 6, 1; [9, 10], [4, 5]; [6, 1, 23, 18]; 6, 1; 27 (003) - example_id = "000021" + example_id = t2m_tgt_skel_id # Lower legs l_idx1, l_idx2 = 5, 8 # Right/Left foot diff --git a/custom_paramUtil.py b/custom_paramUtil.py new file mode 100644 index 0000000..0939aa0 --- /dev/null +++ b/custom_paramUtil.py @@ -0,0 +1,71 @@ +""" +Parameters for the reference skeleton. + +Each skeleton must have: +- kinematic chain: list of lists that reflect joint hierarchy +- raw offsets: np.array of relative positions to parent node in [x, y, z] order +- tgt_skel_id: serial number of the file to read the example skeleton from +""" + +import numpy as np + +# Define a kinematic tree for the skeletal struture +t2m_raw_offsets = np.array([[0,0,0], + [1,0,0], + [-1,0,0], + [0,1,0], + [0,-1,0], + [0,-1,0], + [0,1,0], + [0,-1,0], + [0,-1,0], + [0,1,0], + [0,0,1], + [0,0,1], + [0,1,0], + [1,0,0], + [-1,0,0], + [0,0,1], + [0,-1,0], + [0,-1,0], + [0,-1,0], + [0,-1,0], + [0,-1,0], + [0,-1,0]]) + +t2m_kinematic_chain = [[0, 2, 5, 8, 11], [0, 1, 4, 7, 10], [0, 3, 6, 9, 12, 15], [9, 14, 17, 19, 21], [9, 13, 16, 18, 20]] + +t2m_tgt_skel_id = '000021' + +custom_kinematic_chain = [[0, 1, 2, 3, 4, 5, 6], [1, 7, 8, 9, 10, 11], [1, 12, 13, 14, 15, 16], [13, 17, 18, 19, 20, 21], [13, 22, 23, 24, 25, 26]] +custom_raw_offset = np.array( + [ + [ 0, 0, 0], + [ 0, 1, 0], + [ 1, 0, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [-1, 0, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0, 1, 0], + [ 0, 1, 0], + [ 0, 1, 0], + [ 0, 1, 0], + [ 0, 1, 0], + [-1, 0, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 1, 0, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0], + [ 0,-1, 0] + ] +) \ No newline at end of file From 2a9cead07e103ac6cd4ba169a72f46c7adae15a3 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 08:48:13 +0800 Subject: [PATCH 12/23] Updated build_vector script. - used custom rig instead of regular. --- build_vector.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/build_vector.py b/build_vector.py index 692f969..3e2ef51 100644 --- a/build_vector.py +++ b/build_vector.py @@ -4,7 +4,7 @@ import numpy as np import os from common.quaternion import * -from custom_paramUtil import t2m_kinematic_chain, t2m_raw_offsets, t2m_tgt_skel_id +from custom_paramUtil import custom_kinematic_chain, custom_raw_offsets, custom_tgt_skel_id import torch from tqdm import tqdm @@ -235,26 +235,26 @@ def recover_from_ric(data, joints_num): ## data for existing rig ## orig. 5, 8; [8, 11], [7, 10]; [2, 1, 17, 16]; 2, 1; 22 (000021) ## cf. 6, 1; [9, 10], [4, 5]; [6, 1, 23, 18]; 6, 1; 27 (003) - example_id = t2m_tgt_skel_id + example_id = custom_tgt_skel_id # Lower legs - l_idx1, l_idx2 = 5, 8 + l_idx1, l_idx2 = 6, 1 # Right/Left foot - fid_r, fid_l = [8, 11], [7, 10] + fid_r, fid_l = [9, 10], [4, 5] # Face direction, r_hip, l_hip, sdr_r, sdr_l - face_joint_indx = [2, 1, 17, 16] + face_joint_indx = [6, 1, 23, 18] # l_hip, r_hip - r_hip, l_hip = 2, 1 - joints_num = 22 + r_hip, l_hip = 6, 1 + joints_num = 27 # ds_num = 8 - data_dir = './joints/' - save_dir1 = './HumanML3D/new_joints/' - save_dir2 = './HumanML3D/new_joint_vecs/' + data_dir = './cjoints/' + save_dir1 = './Custom/new_joints/' + save_dir2 = './Custom/new_joint_vecs/' os.makedirs(save_dir1, exist_ok=True) os.makedirs(save_dir2, exist_ok=True) - n_raw_offsets = torch.from_numpy(t2m_raw_offsets) - kinematic_chain = t2m_kinematic_chain + n_raw_offsets = torch.from_numpy(custom_raw_offsets) + kinematic_chain = custom_kinematic_chain # Get offsets of target skeleton example_data = np.load(os.path.join(data_dir, example_id + '.npy')) From f260b571cb1a06107187365f636fd0716352ded8 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 08:49:42 +0800 Subject: [PATCH 13/23] Fixed custom_paramUtil. - renamed variable for import. - added skeleton id for use as example --- custom_paramUtil.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/custom_paramUtil.py b/custom_paramUtil.py index 0939aa0..e4d08f1 100644 --- a/custom_paramUtil.py +++ b/custom_paramUtil.py @@ -38,7 +38,7 @@ t2m_tgt_skel_id = '000021' custom_kinematic_chain = [[0, 1, 2, 3, 4, 5, 6], [1, 7, 8, 9, 10, 11], [1, 12, 13, 14, 15, 16], [13, 17, 18, 19, 20, 21], [13, 22, 23, 24, 25, 26]] -custom_raw_offset = np.array( +custom_raw_offsets = np.array( [ [ 0, 0, 0], [ 0, 1, 0], @@ -68,4 +68,6 @@ [ 0,-1, 0], [ 0,-1, 0] ] -) \ No newline at end of file +) + +custom_tgt_skel_id = "003" \ No newline at end of file From a9a6eb119dcef68c878a67841d5183b7a8bf53f8 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 09:57:13 +0800 Subject: [PATCH 14/23] Added updated and documented annotate_texts.py. - made process_text explicitly return a tuple - made prepare_combined_line explicitly receive a tuple --- annotate_texts.py | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 annotate_texts.py diff --git a/annotate_texts.py b/annotate_texts.py new file mode 100644 index 0000000..6a93c6c --- /dev/null +++ b/annotate_texts.py @@ -0,0 +1,96 @@ +""" +Given a text file with raw descriptions of actions, tag each description with +parts-of-speech tags, then write them in the training format to a new file. +""" + +import spacy + +nlp = spacy.load('en_core_web_sm') + +def process_text(sentence: str) -> tuple[list[str], list[str]]: + """ + Return lists of words and their parts of speech (POS) tags + for a given sentence. + + :param sentence: string to be tagged + :return word_list: list of tokens found in the sentence + :return pos_list: list of part-of-speech tags by token + """ + sentence = sentence.replace('-', '') + doc = nlp(sentence) + word_list = [] + pos_list = [] + for token in doc: + word = token.text + if not word.isalpha(): + continue + if (token.pos_ in ("NOUN", "VERB")) and (word != 'left'): + word_list.append(token.lemma_) + else: + word_list.append(word) + pos_list.append(token.pos_) + return (word_list, pos_list) + + +def read_text_from_file(input_file: str) -> list[str]: + """ + Read the text from a file of action descriptions for parsing. + + :param input_file: string path to the input file + :return result: list of strings read from the input file + """ + with open(input_file, 'r', encoding="utf-8") as infile: + raw_lines = infile.readlines() + result = [line.strip() for line in raw_lines] + + return result + + +def prepare_combined_line(sentence: str, start_time: float=0.0, end_time: float=0.0) -> str: + """ + Given each sentence, parse it and attach tags to each token. + Then include the description start and end time to be edited if needed. + By default, these are 0.0 if we are describing the full sequence. + + :param sentence: string containing an input sentence + :param start_time: float representing start time of the description + :param end_time: float representing end time of the description + :return combined_line: string containing sentence#tagged_sentence#start_time#end_time + """ + (words, tags) = process_text(sentence) + tagged_sentence = ' '.join([f"{n[0]}/{n[1]}" for n in zip(words, tags)]) + combined_line = f"{sentence}#{tagged_sentence}#{start_time}#{end_time}\n" + + return combined_line + + +def write_output_file(output_list: list[str], output_file: str): + """ + Write a specified output list to a specified file. + + :param output_list: list of strings to write + :param output_file: string path to output file location + """ + with open(output_file, 'w', encoding="utf-8") as outfile: + outfile.writelines(output_list) + + +def main(): + """ + Actually run the conversion here. + """ + output = [] + strings = [] + INPUT_FILE = "testfile.txt" + OUTPUT_FILE = "taggedfile.txt" + + strings = read_text_from_file(input_file=INPUT_FILE) + + for input_line in strings: + output_line = prepare_combined_line(input_line) + output.append(output_line) + + write_output_file(output_list=output, output_file=OUTPUT_FILE) + +if __name__ == "__main__": + main() From 73e69be0e2e7ec7525bef18d9d05ca5e5400de25 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 10:06:08 +0800 Subject: [PATCH 15/23] Ignored content from unzipping downloads where instructed. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 130b8b7..778b077 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ # HumanML3D texts, when unzipped HumanML3D/texts/*.txt +# amass or body model data, when unzipped +amass_data/ +body_models/ + # zip files *.zip *.bz2 From d1efe48b768bb2c9e0404223743d05ac19e5a9b7 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 10:42:19 +0800 Subject: [PATCH 16/23] Added documented version of mean/variance calculations. --- cal_mean_variance.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 cal_mean_variance.py diff --git a/cal_mean_variance.py b/cal_mean_variance.py new file mode 100644 index 0000000..a4b12c4 --- /dev/null +++ b/cal_mean_variance.py @@ -0,0 +1,56 @@ +import numpy as np +import os +from os.path import join as pjoin + + +# root_rot_velocity (B, seq_len, 1) +# root_linear_velocity (B, seq_len, 2) +# root_y (B, seq_len, 1) +# ric_data (B, seq_len, (joint_num - 1)*3) +# rot_data (B, seq_len, (joint_num - 1)*6) +# local_velocity (B, seq_len, joint_num*3) +# foot contact (B, seq_len, 4) +def mean_variance(data_dir, save_dir, joints_num): + file_list = os.listdir(data_dir) + data_list = [] + + for file in file_list: + data = np.load(pjoin(data_dir, file)) + if np.isnan(data).any(): + print(file) + continue + data_list.append(data) + + data = np.concatenate(data_list, axis=0) + print(data.shape) + # each frame is described in four broad sections given a rig with j joints: + # 0 -- 3 : global position, 4 + # 4 -- 4 + (j - 1) * 3 : local positions of other joints with respect to root, (j-1) * 3 + # 4 + (j - 1) * 3 -- 4 + (j - 1) * 9 : local rotations of other joints with respect to root, (j-1) * 6 + # 4 + (j - 1) * 9 -- end : global joint velocities plus foot contact information, j * 3 + 4 + # see https://arxiv.org/pdf/2405.11126 Appendix A for more details + section_one = 4 + (joints_num - 1) * 3 + section_two = 4 + (joints_num - 1) * 9 + section_three = section_two + joints_num * 3 + Mean = data.mean(axis=0) + Std = data.std(axis=0) + Std[0:1] = Std[0:1].mean() / 1.0 + Std[1:3] = Std[1:3].mean() / 1.0 + Std[3:4] = Std[3:4].mean() / 1.0 + Std[4: section_one] = Std[4: section_one].mean() / 1.0 + Std[section_one: section_two] = Std[section_one: section_two].mean() / 1.0 + Std[section_two: section_three] = Std[section_two: section_three].mean() / 1.0 + Std[section_three: ] = Std[section_three: ].mean() / 1.0 + + assert 8 + (joints_num - 1) * 9 + joints_num * 3 == Std.shape[-1] + + np.save(pjoin(save_dir, 'Mean_abs_3d.npy'), Mean) + np.save(pjoin(save_dir, 'Std_abs_3d.npy'), Std) + + return Mean, Std + +if __name__ == '__main__': + data_dir = './HumanML3D/new_joint_vecs_abs_3d/' + save_dir = './HumanML3D/' + mean, std = mean_variance(data_dir, save_dir, 22) + From 7faf1d5a0c203ec8aa99933ee688f54f2a1ba00e Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 11:06:17 +0800 Subject: [PATCH 17/23] Updated docstrings and rearranged imports. --- cal_mean_variance.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/cal_mean_variance.py b/cal_mean_variance.py index a4b12c4..ca07728 100644 --- a/cal_mean_variance.py +++ b/cal_mean_variance.py @@ -1,8 +1,13 @@ -import numpy as np -import os -from os.path import join as pjoin - +""" +Calculate the mean and variance of sequence vectors for a given rig type. +# each frame is described in four broad sections given a rig with j joints: +# 0 -- 3 : global position, 4 +# 4 -- 4 + (j - 1) * 3 : local positions of other joints with respect to root, (j - 1) * 3 +# 4 + (j - 1) * 3 -- 4 + (j - 1) * 9 : local rotations of other joints with respect to root, (j - 1) * 6 +# 4 + (j - 1) * 9 -- end : global joint velocities plus foot contact information, j * 3 + 4 +# see https://arxiv.org/pdf/2405.11126 Appendix A for more details +# from the computed data, this corresponds to: # root_rot_velocity (B, seq_len, 1) # root_linear_velocity (B, seq_len, 2) # root_y (B, seq_len, 1) @@ -10,7 +15,21 @@ # rot_data (B, seq_len, (joint_num - 1)*6) # local_velocity (B, seq_len, joint_num*3) # foot contact (B, seq_len, 4) -def mean_variance(data_dir, save_dir, joints_num): + +""" +import os +from os.path import join as pjoin +import numpy as np + + +def mean_variance(data_dir: str, save_dir: str, joints_num: int): + """ + Compute the mean and variance of the joint vector sequences in a given directory. + + :param data_dir: string path to the vectors whose mean and variance will be computed + :param save_dir: string path to the directory where the results will be stored + :param joints_num: integer number of joints for a given rig type + """ file_list = os.listdir(data_dir) data_list = [] @@ -23,12 +42,6 @@ def mean_variance(data_dir, save_dir, joints_num): data = np.concatenate(data_list, axis=0) print(data.shape) - # each frame is described in four broad sections given a rig with j joints: - # 0 -- 3 : global position, 4 - # 4 -- 4 + (j - 1) * 3 : local positions of other joints with respect to root, (j-1) * 3 - # 4 + (j - 1) * 3 -- 4 + (j - 1) * 9 : local rotations of other joints with respect to root, (j-1) * 6 - # 4 + (j - 1) * 9 -- end : global joint velocities plus foot contact information, j * 3 + 4 - # see https://arxiv.org/pdf/2405.11126 Appendix A for more details section_one = 4 + (joints_num - 1) * 3 section_two = 4 + (joints_num - 1) * 9 section_three = section_two + joints_num * 3 From d44dcfffafedfe7ec57f190599a2aa7593402548 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 11:07:44 +0800 Subject: [PATCH 18/23] Added changes needed for custom rig. --- cal_mean_variance.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cal_mean_variance.py b/cal_mean_variance.py index ca07728..1cdc445 100644 --- a/cal_mean_variance.py +++ b/cal_mean_variance.py @@ -63,7 +63,6 @@ def mean_variance(data_dir: str, save_dir: str, joints_num: int): return Mean, Std if __name__ == '__main__': - data_dir = './HumanML3D/new_joint_vecs_abs_3d/' - save_dir = './HumanML3D/' - mean, std = mean_variance(data_dir, save_dir, 22) - + data_dir = './Custom/new_joint_vecs/' + save_dir = './Custom/' + mean, std = mean_variance(data_dir, save_dir, 27) From ab10ede75985a63cae4f1b2ddf926a5326a8df89 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 21 Nov 2024 12:08:51 +0800 Subject: [PATCH 19/23] Upgraded annotate_texts. - refactored single file annotation out - added main function which deals with directories --- annotate_texts.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/annotate_texts.py b/annotate_texts.py index 6a93c6c..113cfa5 100644 --- a/annotate_texts.py +++ b/annotate_texts.py @@ -4,6 +4,7 @@ """ import spacy +from tqdm import tqdm nlp = spacy.load('en_core_web_sm') @@ -75,22 +76,39 @@ def write_output_file(output_list: list[str], output_file: str): outfile.writelines(output_list) -def main(): +def tag_one_file(input_file: str, output_file: str): """ - Actually run the conversion here. + Do the tagging for one input file and return one output file. + + :param input_file: string path to file with untagged descriptions + :param output_file: string path to file for storing results """ output = [] strings = [] - INPUT_FILE = "testfile.txt" - OUTPUT_FILE = "taggedfile.txt" - strings = read_text_from_file(input_file=INPUT_FILE) + strings = read_text_from_file(input_file=input_file) for input_line in strings: output_line = prepare_combined_line(input_line) output.append(output_line) - write_output_file(output_list=output, output_file=OUTPUT_FILE) + write_output_file(output_list=output, output_file=output_file) + + +def main(): + """ + Allow for directories to be passed in + """ + from os import listdir + from os.path import join as pjoin + + data_dir = "Custom/texts/raw" + save_dir = "Custom/texts" + + for text_file in tqdm(listdir(data_dir)): + print(f"Processing {text_file}...") + tag_one_file(input_file=pjoin(data_dir, text_file), output_file=pjoin(save_dir, text_file)) + if __name__ == "__main__": main() From 560041554afaa857850021f4313eea1ecf6552cd Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 19 Dec 2024 14:57:43 +0800 Subject: [PATCH 20/23] Added names to arguments in cal_mean_variance for clarity. --- cal_mean_variance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cal_mean_variance.py b/cal_mean_variance.py index 1cdc445..76ddcce 100644 --- a/cal_mean_variance.py +++ b/cal_mean_variance.py @@ -65,4 +65,4 @@ def mean_variance(data_dir: str, save_dir: str, joints_num: int): if __name__ == '__main__': data_dir = './Custom/new_joint_vecs/' save_dir = './Custom/' - mean, std = mean_variance(data_dir, save_dir, 27) + mean, std = mean_variance(data_dir=data_dir, save_dir=save_dir, joints_num=27) From d5b04b93fd5e182c399bf7d0b4c27fde2baaada8 Mon Sep 17 00:00:00 2001 From: icedwater Date: Thu, 19 Dec 2024 14:59:50 +0800 Subject: [PATCH 21/23] Added processed texts to .gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 778b077..e3d3b28 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ # HumanML3D texts, when unzipped HumanML3D/texts/*.txt +# Custom texts, once processed +Custom/texts + # amass or body model data, when unzipped amass_data/ body_models/ From 2591e488a4c18e5273255a15868e047dae77d2e6 Mon Sep 17 00:00:00 2001 From: icedwater Date: Fri, 7 Mar 2025 16:33:24 +0800 Subject: [PATCH 22/23] Added precalculate script for convenience. --- precalculate | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 precalculate diff --git a/precalculate b/precalculate new file mode 100755 index 0000000..f5af59b --- /dev/null +++ b/precalculate @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +FILE="$1" + +joint_names=() + +# add root +joint_names=("${joint_names[@]}" "\"$(grep ROOT ${FILE} | sed 's/ROOT //;s/\t//g')\",") + +# add all joints in order +for joint in $(grep JOINT ${FILE} | sed 's/JOINT //;s/\t//g'); do + joint_names+=( "\"${joint}\"," ); # FIXME: hacky method to include quotes and comma +done + +# remove comma from last joint +last_joint=${joint_names[-1]} +joint_names[-1]=${last_joint%,*} + +# convenience computations +num_joints=${#joint_names[@]} +num_params=$(( ${num_joints} * 12 - 1 )) + +# write outputs to file +echo "From ${FILE}:" +echo "Joints = ${num_joints}" +echo "Params = ${num_params} (12 * Joints - 1)" + +echo +echo "Use the following as RIG_JOINT_NAMES:" +echo "RIG_JOINT_NAMES = [" +for joint in "${joint_names[@]}"; do echo ${joint}; done +echo "]" From 74c9ba29fb7ebc4c1c4573f4e25c8324de4086a2 Mon Sep 17 00:00:00 2001 From: icedwater Date: Fri, 7 Mar 2025 16:52:00 +0800 Subject: [PATCH 23/23] Added requirements.txt and relevant line in README. --- README.md | 2 ++ requirements.txt | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 4ddaa47..0d04534 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ conda env create -f environment.yaml conda activate torch_render ``` +Alternatively, install `requirements.txt` into the virtual environment using the workflow of your choice. + In the case of installation failure, you could alternatively install the following: ```sh - Python==3.7.10 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c8da00 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +--extra-index-url https://download.pytorch.org/whl/cu113 +torch==1.12.1+cu113 # for this setup +numpy==1.21.0 +scipy==1.7.3 +tqdm==4.66.5 +pandas==1.3.5 +spacy>=3.7 # for text processing +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl