File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -117,7 +117,7 @@ def loss_ownership_corr(
117
117
# So we need to multiply vectors x by a factor of tanh(|x|) / |x| = tanh(sqrt(|x|^2)) / sqrt(|x|^2)
118
118
# But there's a division by 0 when |x| = 0, and also sqrt(0) has infinite gradient..
119
119
# So to do this in a numerically stable way, we do this piecewise, using 3rd order taylor expansion
120
- # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/105 x^3.
120
+ # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/315 x^3.
121
121
delta = 0.010
122
122
sqrtmagsqboundedbelow = torch .sqrt (torch .clamp (magsq ,min = 0.008 ))
123
123
magsqboundedabove = torch .clamp (magsq ,max = 0.012 )
@@ -159,7 +159,7 @@ def loss_futurepos_corr(
159
159
# So we need to multiply vectors x by a factor of tanh(|x|) / |x| = tanh(sqrt(|x|^2)) / sqrt(|x|^2)
160
160
# But there's a division by 0 when |x| = 0, and also sqrt(0) has infinite gradient..
161
161
# So to do this in a numerically stable way, we do this piecewise, using 3rd order taylor expansion
162
- # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/105 x^3.
162
+ # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/315 x^3.
163
163
delta = 0.010
164
164
sqrtmagsqboundedbelow = torch .sqrt (torch .clamp (magsq ,min = 0.008 ))
165
165
magsqboundedabove = torch .clamp (magsq ,max = 0.012 )
You can’t perform that action at this time.
0 commit comments