Skip to content

Commit 850a57d

Browse files
byshiuebyshiue
authored
feat: Add v1.0 codes (#1)
1 parent b3ebeb9 commit 850a57d

16 files changed

+2486
-1
lines changed

‎.clang-format‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
BasedOnStyle: Google
3+
4+
IndentWidth: 2
5+
ContinuationIndentWidth: 4
6+
UseTab: Never
7+
MaxEmptyLinesToKeep: 2
8+
9+
SortIncludes: true
10+
CompactNamespaces: true
11+
ReflowComments: true
12+
13+
DerivePointerAlignment: false
14+
PointerAlignment: Left
15+
16+
AllowShortIfStatementsOnASingleLine: false
17+
AllowShortBlocksOnASingleLine: false
18+
AllowShortFunctionsOnASingleLine: Inline
19+
20+
AlwaysBreakAfterReturnType: TopLevelDefinitions
21+
AlignAfterOpenBracket: AlwaysBreak
22+
BreakBeforeBraces: Custom
23+
BraceWrapping:
24+
AfterClass: false
25+
AfterControlStatement: false
26+
AfterEnum: false
27+
AfterFunction: true
28+
AfterNamespace: false
29+
AfterStruct: false
30+
AfterUnion: false
31+
BeforeCatch: true
32+
33+
BinPackArguments: true
34+
BinPackParameters: true
35+
ConstructorInitializerAllOnOneLineOrOnePerLine: false
36+
37+
IndentCaseLabels: true

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/build
2+
/.vscode
3+
*.so
4+
*.run
5+
.clangd
6+
compile_commands.json
7+
../all_models/transformer/1/*

‎CMakeLists.txt‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
cmake_minimum_required (VERSION 3.18)
28+
29+
project(tritontransformerbackend LANGUAGES C CXX)
30+
31+
#
32+
# Options
33+
#
34+
option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON)
35+
option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)
36+
set(TRITON_PYTORCH_INCLUDE_PATHS "" CACHE PATH "Paths to Torch includes")
37+
set(TRITON_PYTORCH_LIB_PATHS "" CACHE PATH "Paths to Torch libraries")
38+
39+
set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo")
40+
set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo")
41+
set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo")
42+
43+
if(NOT CMAKE_BUILD_TYPE)
44+
set(CMAKE_BUILD_TYPE Release)
45+
endif()
46+
47+
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
48+
49+
# Python.h needed by torch headers.
50+
find_package(Python3 REQUIRED COMPONENTS Development)
51+
52+
find_package(FasterTransformer)
53+
find_package(CUDA 10.1 REQUIRED)
54+
find_package(MPI REQUIRED)
55+
find_package(NCCL REQUIRED)
56+
57+
message(STATUS "Found MPI (include: ${MPI_INCLUDE_DIRS}, library: ${MPI_LIBRARIES})")
58+
59+
if (${CUDA_VERSION} GREATER_EQUAL 11.0)
60+
message(STATUS "Add DCUDA11_MODE")
61+
add_definitions("-DCUDA11_MODE")
62+
endif()
63+
64+
#
65+
# Dependencies
66+
#
67+
# FetchContent's composibility isn't very good. We must include the
68+
# transitive closure of all repos so that we can override the tag.
69+
#
70+
include(FetchContent)
71+
72+
FetchContent_Declare(
73+
repo-common
74+
GIT_REPOSITORY https://github.com/triton-inference-server/common.git
75+
GIT_TAG ${TRITON_COMMON_REPO_TAG}
76+
GIT_SHALLOW ON
77+
)
78+
FetchContent_Declare(
79+
repo-core
80+
GIT_REPOSITORY https://github.com/triton-inference-server/core.git
81+
GIT_TAG ${TRITON_CORE_REPO_TAG}
82+
GIT_SHALLOW ON
83+
)
84+
FetchContent_Declare(
85+
repo-backend
86+
GIT_REPOSITORY https://github.com/triton-inference-server/backend.git
87+
GIT_TAG ${TRITON_BACKEND_REPO_TAG}
88+
GIT_SHALLOW ON
89+
)
90+
FetchContent_Declare(
91+
repo-ft
92+
GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer.git
93+
GIT_TAG main
94+
GIT_SHALLOW ON
95+
)
96+
FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft)
97+
98+
#
99+
# CUDA
100+
#
101+
if(${TRITON_ENABLE_GPU})
102+
find_package(CUDAToolkit REQUIRED)
103+
endif() # TRITON_ENABLE_GPU
104+
105+
#
106+
# Shared library implementing the Triton Backend API
107+
#
108+
configure_file(src/libtriton_transformer.ldscript libtriton_transformer.ldscript COPYONLY)
109+
110+
add_library(
111+
triton-transformer-backend SHARED
112+
src/libtransformer.cc
113+
)
114+
115+
add_library(
116+
TritonTransformerBackend::triton-transformer-backend ALIAS triton-transformer-backend
117+
)
118+
119+
#find_package(CUDAToolkit REQUIRED)
120+
find_package(CUDA 10.1 REQUIRED)
121+
find_package(MPI REQUIRED)
122+
##find_package(NCCL REQUIRED)
123+
#if (${CUDA_VERSION} GREATER_EQUAL 11.0)
124+
message(STATUS "Add DCUDA11_MODE")
125+
add_definitions("-DCUDA11_MODE")
126+
#endif()
127+
128+
set(CUDA_PATH ${CUDA_TOOLKIT_ROOT_DIR})
129+
130+
target_compile_definitions(triton-transformer-backend
131+
PUBLIC
132+
USE_TRITONSERVER_DATATYPE
133+
BUILD_GPT)
134+
135+
target_include_directories(
136+
triton-transformer-backend
137+
PRIVATE
138+
${CMAKE_CURRENT_SOURCE_DIR}/src
139+
# ${CMAKE_CURRENT_SOURCE_DIR}/test
140+
${TRITON_PYTORCH_INCLUDE_PATHS}
141+
${Python3_INCLUDE_DIRS}
142+
${MPI_INCLUDE_PATH}
143+
${repo-ft_SOURCE_DIR}
144+
)
145+
146+
target_link_directories(
147+
triton-transformer-backend
148+
PRIVATE
149+
${CUDA_PATH}/lib64
150+
${MPI_Libraries}
151+
)
152+
153+
target_compile_features(triton-transformer-backend PRIVATE cxx_std_14)
154+
target_compile_options(
155+
triton-transformer-backend PRIVATE
156+
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
157+
-Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
158+
)
159+
160+
if(${TRITON_ENABLE_GPU})
161+
target_compile_definitions(
162+
triton-transformer-backend
163+
PRIVATE TRITON_ENABLE_GPU=1
164+
)
165+
endif() # TRITON_ENABLE_GPU
166+
167+
set_target_properties(
168+
triton-transformer-backend
169+
PROPERTIES
170+
POSITION_INDEPENDENT_CODE ON
171+
OUTPUT_NAME triton_transformer
172+
SKIP_BUILD_RPATH TRUE
173+
BUILD_WITH_INSTALL_RPATH TRUE
174+
INSTALL_RPATH_USE_LINK_PATH FALSE
175+
INSTALL_RPATH "$\{ORIGIN\}"
176+
LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_transformer.ldscript
177+
LINK_FLAGS "-Wl,--no-as-needed,--version-script libtriton_transformer.ldscript"
178+
)
179+
180+
# Need to turn off unused-but-set-variable due to Torchvision
181+
# Need to turn off unknown-pragmas due to ATen OpenMP
182+
set_target_properties(
183+
triton-transformer-backend
184+
PROPERTIES COMPILE_FLAGS
185+
"-Wno-unknown-pragmas -Wno-unused-but-set-variable"
186+
)
187+
188+
set(TRITON_PYTORCH_LDFLAGS "")
189+
FOREACH(p ${TRITON_PYTORCH_LIB_PATHS})
190+
set(TRITON_PYTORCH_LDFLAGS ${TRITON_PYTORCH_LDFLAGS} "-L${p}")
191+
ENDFOREACH(p)
192+
193+
target_link_libraries(
194+
triton-transformer-backend
195+
PRIVATE
196+
triton-core-serverapi # from repo-core
197+
triton-core-backendapi # from repo-core
198+
triton-core-serverstub # from repo-core
199+
triton-backend-utils # from repo-backend
200+
transformer-shared # from repo-ft
201+
${TRITON_PYTORCH_LDFLAGS}
202+
${NCCL_LIBRARIES}
203+
${MPI_LIBRARIES}
204+
#-ltorch
205+
#-ltorch_cpu
206+
#-ltorch_cuda
207+
#-ltorchvision
208+
#-lc10
209+
#-lc10_cuda
210+
#-lmkl_core
211+
#-lmkl_gnu_thread
212+
#-lmkl_intel_lp64
213+
#-lmkl_avx2
214+
#-lmkl_def
215+
#-liomp5
216+
#-lmkl_intel_thread
217+
#-lmkl_vml_def
218+
#-lmkl_rt
219+
-lcublas
220+
-lcublasLt
221+
-lcudart
222+
-lcurand
223+
#-lnccl
224+
#-lmpi
225+
)
226+
227+
if(${TRITON_ENABLE_GPU})
228+
target_link_libraries(
229+
triton-transformer-backend
230+
PRIVATE
231+
CUDA::cudart
232+
)
233+
endif() # TRITON_ENABLE_GPU
234+
235+
#
236+
# Install
237+
#
238+
include(GNUInstallDirs)
239+
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonTransformerBackend)
240+
241+
install(
242+
TARGETS
243+
triton-transformer-backend
244+
EXPORT
245+
triton-transformer-backend-targets
246+
LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/transformer
247+
ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/transformer
248+
)
249+
250+
install(
251+
EXPORT
252+
triton-transformer-backend-targets
253+
FILE
254+
TritonTransformerBackendTargets.cmake
255+
NAMESPACE
256+
TritonTransformerBackend::
257+
DESTINATION
258+
${INSTALL_CONFIGDIR}
259+
)
260+
261+
include(CMakePackageConfigHelpers)
262+
configure_package_config_file(
263+
${CMAKE_CURRENT_LIST_DIR}/cmake/TritonTransformerBackendConfig.cmake.in
264+
${CMAKE_CURRENT_BINARY_DIR}/TritonTransformerBackendConfig.cmake
265+
INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
266+
)
267+
268+
install(
269+
FILES
270+
${CMAKE_CURRENT_BINARY_DIR}/TritonTransformerBackendConfig.cmake
271+
DESTINATION ${INSTALL_CONFIGDIR}
272+
)
273+
274+
#
275+
# Export from build tree
276+
#
277+
export(
278+
EXPORT triton-transformer-backend-targets
279+
FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonTransformerBackendTargets.cmake
280+
NAMESPACE TritonTransformerBackend::
281+
)
282+
283+
export(PACKAGE TritonTransformerBackend)

0 commit comments

Comments
 (0)