Skip to content

Commit 8bfa21f

Browse files
vipnydavvipnydav
committed
test(read_cache): bifurcate read_cache local modification tests for rapid writes
1 parent 5716f09 commit 8bfa21f

3 files changed

Lines changed: 100 additions & 38 deletions

File tree

‎tools/integration_tests/read_cache/local_modification_test.go‎

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,11 @@ import (
3434
// Boilerplate
3535
// //////////////////////////////////////////////////////////////////////
3636
type localModificationTest struct {
37-
flags []string
38-
storageClient *storage.Client
39-
ctx context.Context
40-
baseTestName string
37+
flags []string
38+
storageClient *storage.Client
39+
ctx context.Context
40+
baseTestName string
41+
isRapidWritesEnabled bool
4142
suite.Suite
4243
}
4344

@@ -75,14 +76,13 @@ func (s *localModificationTest) TestReadAfterLocalGCSFuseWriteIsCacheMiss() {
7576
expectedOutcome1 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize, true, s.T())
7677
// Append data in the same file to change object generation.
7778
smallContent, err := operations.GenerateRandomData(smallContentSize)
78-
if err != nil {
79-
s.T().Errorf("TestReadAfterLocalGCSFuseWriteIsCacheMiss: could not generate randomm data: %v", err)
80-
}
79+
require.NoError(s.T(), err, "TestReadAfterLocalGCSFuseWriteIsCacheMiss: could not generate random data")
80+
8181
err = operations.WriteFileInAppendMode(path.Join(testEnv.testDirPath, testFileName), string(smallContent))
82-
if err != nil {
83-
s.T().Errorf("Error in appending data in file: %v", err)
84-
}
85-
if !setup.IsZonalBucketRun() {
82+
require.NoError(s.T(), err, "Error in appending data in file")
83+
84+
isPirloRapidWrites := setup.IsPirloBucketRun() && s.isRapidWritesEnabled
85+
if !setup.IsZonalBucketRun() && !isPirloRapidWrites {
8686
// Read file 2nd time.
8787
expectedOutcome2 := readFileAndValidateCacheWithGCS(s.ctx, s.storageClient, testFileName, fileSize+smallContentSize, true, s.T())
8888

@@ -113,19 +113,38 @@ func (s *localModificationTest) TestReadAfterLocalGCSFuseWriteIsCacheMiss() {
113113
// Test Function (Runs once before all tests)
114114
////////////////////////////////////////////////////////////////////////
115115

116-
func TestLocalModificationTest(t *testing.T) {
117-
ts := &localModificationTest{ctx: context.Background(), storageClient: testEnv.storageClient, baseTestName: t.Name()}
118-
119-
// Run tests for mounted directory if the flag is set. This assumes that run flag is properly passed by GKE team as per the config.
116+
func runLocalModificationTest(t *testing.T, ts *localModificationTest) {
120117
if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" {
121118
suite.Run(t, ts)
122119
return
123120
}
124121

125-
// Run tests for GCE environment otherwise.
126122
flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name())
127123
for _, ts.flags = range flagsSet {
128-
log.Printf("Running tests with flags: %s", ts.flags)
124+
log.Printf("Running %s with flags: %s", t.Name(), ts.flags)
129125
suite.Run(t, ts)
130126
}
131127
}
128+
129+
func TestLocalModificationBase(t *testing.T) {
130+
ts := &localModificationTest{
131+
ctx: context.Background(),
132+
storageClient: testEnv.storageClient,
133+
baseTestName: t.Name(),
134+
isRapidWritesEnabled: false,
135+
}
136+
runLocalModificationTest(t, ts)
137+
}
138+
139+
func TestLocalModificationRapidWritesEnabled(t *testing.T) {
140+
if !setup.IsPirloBucketRun() {
141+
t.Skip("Rapid writes tests are only applicable to Pirlo buckets")
142+
}
143+
ts := &localModificationTest{
144+
ctx: context.Background(),
145+
storageClient: testEnv.storageClient,
146+
baseTestName: t.Name(),
147+
isRapidWritesEnabled: true,
148+
}
149+
runLocalModificationTest(t, ts)
150+
}

‎tools/integration_tests/read_cache/setup_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,13 +172,13 @@ func TestMain(m *testing.M) {
172172
cfg.ReadCache[0].Configs[3].Run = "TestRangeReadWithParallelDownloadsTest"
173173

174174
cfg.ReadCache[0].Configs[4].Flags = []string{
175-
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false",
176-
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --enable-kernel-reader=false",
177-
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --implicit-dirs --client-protocol=grpc --enable-kernel-reader=false",
178-
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationTest --log-file=/gcsfuse-tmp/TestLocalModificationTest.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false",
175+
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationBase --log-file=/gcsfuse-tmp/TestLocalModificationBase.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false",
176+
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationBase --log-file=/gcsfuse-tmp/TestLocalModificationBase.log --log-severity=TRACE --enable-kernel-reader=false",
177+
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestLocalModificationBase --log-file=/gcsfuse-tmp/TestLocalModificationBase.log --log-severity=TRACE --implicit-dirs --client-protocol=grpc --enable-kernel-reader=false",
178+
"--file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=true --cache-dir=/gcsfuse-tmp/TestLocalModificationBase --log-file=/gcsfuse-tmp/TestLocalModificationBase.log --log-severity=TRACE --client-protocol=grpc --enable-kernel-reader=false",
179179
}
180180
cfg.ReadCache[0].Configs[4].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true}
181-
cfg.ReadCache[0].Configs[4].Run = "TestLocalModificationTest"
181+
cfg.ReadCache[0].Configs[4].Run = "TestLocalModificationBase"
182182

183183
cfg.ReadCache[0].Configs[5].Flags = []string{
184184
"--stat-cache-ttl=0s --file-cache-max-size-mb=9 --file-cache-enable-parallel-downloads=false --cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest --log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log --log-severity=TRACE --implicit-dirs --enable-kernel-reader=false",

‎tools/integration_tests/test_config.yaml‎

Lines changed: 59 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,31 @@ implicit_dir:
3636
hns: true
3737
zonal: true
3838
run_on_gke: true
39+
run: TestImplicitDirBase
3940
- flags:
40-
- "--experimental-enable-pirlo,--implicit-dirs,--client-protocol=http1"
41+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--implicit-dirs"
4142
run_on_pirlo:
4243
hns:
4344
same_zone: true
4445
different_zone: false
45-
run_on_gke: true
46+
run_on_gke: false
47+
run: TestImplicitDirLocalFileRapidWritesEnabled
48+
- flags:
49+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--implicit-dirs"
50+
run_on_pirlo:
51+
hns:
52+
same_zone: true
53+
different_zone: false
54+
run_on_gke: false
55+
run: TestImplicitDirBase
4656
- flags:
4757
- "--implicit-dirs,--client-protocol=grpc"
4858
compatible:
4959
flat: true
5060
hns: true
5161
zonal: false
5262
run_on_gke: true
63+
run: TestImplicitDirBase
5364

5465
list_large_dir:
5566
- mounted_directory: "${MOUNTED_DIR}"
@@ -111,14 +122,25 @@ operations:
111122
hns: true
112123
zonal: true
113124
run_on_gke: true
125+
run: TestOperationsBase
114126
- flags:
115-
- "--experimental-enable-pirlo,--metadata-cache-ttl-secs=0,--enable-streaming-writes=false,--client-protocol=http1"
116-
- "--experimental-enable-pirlo,--kernel-list-cache-ttl-secs=-1,--implicit-dirs,--enable-metadata-prefetch,--client-protocol=http1"
127+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--metadata-cache-ttl-secs=0,--enable-streaming-writes=false"
128+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--kernel-list-cache-ttl-secs=-1,--implicit-dirs,--enable-metadata-prefetch"
117129
run_on_pirlo:
118130
hns:
119131
same_zone: true
120132
different_zone: false
121-
run_on_gke: true
133+
run_on_gke: false
134+
run: TestWriteOperationsRapidWritesEnabled
135+
- flags:
136+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--metadata-cache-ttl-secs=0,--enable-streaming-writes=false"
137+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--kernel-list-cache-ttl-secs=-1,--implicit-dirs,--enable-metadata-prefetch"
138+
run_on_pirlo:
139+
hns:
140+
same_zone: true
141+
different_zone: false
142+
run_on_gke: false
143+
run: TestOperationsBase
122144
- flags:
123145
- "--enable-atomic-rename-object=true,--client-protocol=http1"
124146
- "--experimental-enable-json-read=true,--client-protocol=http1"
@@ -130,6 +152,7 @@ operations:
130152
zonal: true
131153
tpc: true
132154
run_on_gke: false
155+
run: TestOperationsBase
133156

134157
write_large_files:
135158
- mounted_directory: "${MOUNTED_DIR}"
@@ -450,25 +473,34 @@ read_cache:
450473
run_on_gke: true
451474
run: TestRangeReadWithParallelDownloadsTest
452475
- flags:
453-
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false,--client-protocol=http1"
454-
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--enable-kernel-reader=false,--client-protocol=http1"
455-
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false"
456-
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false"
476+
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false,--client-protocol=http1"
477+
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--enable-kernel-reader=false,--client-protocol=http1"
478+
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--implicit-dirs,--client-protocol=grpc,--enable-kernel-reader=false"
479+
- "--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--client-protocol=grpc,--enable-kernel-reader=false"
457480
compatible:
458481
flat: true
459482
hns: true
460483
zonal: true
461-
run: TestLocalModificationTest
484+
run: TestLocalModificationBase
462485
run_on_gke: true
463486
- flags:
464-
- "--experimental-enable-pirlo,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false,--client-protocol=http1"
465-
- "--experimental-enable-pirlo,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationTest,--log-file=/gcsfuse-tmp/TestLocalModificationTest.log,--log-severity=TRACE,--enable-kernel-reader=false,--client-protocol=http1"
487+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationRapidWritesEnabled,--log-file=/gcsfuse-tmp/TestLocalModificationRapidWritesEnabled.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false"
488+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationRapidWritesEnabled,--log-file=/gcsfuse-tmp/TestLocalModificationRapidWritesEnabled.log,--log-severity=TRACE,--enable-kernel-reader=false"
466489
run_on_pirlo:
467490
hns:
468491
same_zone: true
469492
different_zone: false
470-
run_on_gke: true
471-
run: TestLocalModificationTest
493+
run_on_gke: false
494+
run: TestLocalModificationRapidWritesEnabled
495+
- flags:
496+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false"
497+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestLocalModificationBase,--log-file=/gcsfuse-tmp/TestLocalModificationBase.log,--log-severity=TRACE,--enable-kernel-reader=false"
498+
run_on_pirlo:
499+
hns:
500+
same_zone: true
501+
different_zone: false
502+
run_on_gke: false
503+
run: TestLocalModificationBase
472504
- flags:
473505
- "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads=false,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--implicit-dirs,--enable-kernel-reader=false,--client-protocol=http1"
474506
- "--stat-cache-ttl=0s,--file-cache-max-size-mb=9,--file-cache-enable-parallel-downloads,--cache-dir=/gcsfuse-tmp/TestDisabledCacheTTLTest,--log-file=/gcsfuse-tmp/TestDisabledCacheTTLTest.log,--log-severity=TRACE,--enable-kernel-reader=false,--client-protocol=http1"
@@ -1134,14 +1166,25 @@ readonly_creds:
11341166
hns: true
11351167
zonal: true
11361168
run_on_gke: false
1169+
run: TestReadOnlyCredsBase
1170+
- flags:
1171+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--implicit-dirs=true"
1172+
- "--experimental-enable-pirlo,--enable-rapid-writes=true,--implicit-dirs=false"
1173+
run_on_pirlo:
1174+
hns:
1175+
same_zone: true
1176+
different_zone: false
1177+
run_on_gke: false
1178+
run: TestReadOnlyCredsRapidWritesEnabled
11371179
- flags:
1138-
- "--experimental-enable-pirlo,--implicit-dirs=true,--client-protocol=http1"
1139-
- "--experimental-enable-pirlo,--implicit-dirs=false,--client-protocol=http1"
1180+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--implicit-dirs=true"
1181+
- "--experimental-enable-pirlo,--enable-rapid-writes=false,--implicit-dirs=false"
11401182
run_on_pirlo:
11411183
hns:
11421184
same_zone: true
11431185
different_zone: false
11441186
run_on_gke: false
1187+
run: TestReadOnlyCredsBase
11451188

11461189
mount_timeout:
11471190
- mounted_directory: "${MOUNTED_DIR}"

0 commit comments

Comments
 (0)