Skip to content

Commit fd87724

Browse files
ajoymajumdarajoymajumdar
authored
Overloaded getCatalog API with another optimized API. (Netflix#304)
* Overloaded getCatalog API with another optimized API that addeds the list of database names and user metadata only if explicitly requested. * Upgraded iceberg versions to 0.3.2.
1 parent ef1c6dd commit fd87724

File tree

8 files changed

+101
-16
lines changed

8 files changed

+101
-16
lines changed

‎build.gradle

+3
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ configure(javaProjects) {
150150
dependency("org.elasticsearch.client:transport:5.4.1")
151151
dependency("net.snowflake:snowflake-jdbc:3.4.2")
152152
dependency("com.esotericsoftware.kryo:kryo:2.22")
153+
dependency("com.github.Netflix.iceberg:iceberg-common:0.3.2")
154+
dependency("com.github.Netflix.iceberg:iceberg-core:0.3.2")
155+
dependency("com.github.Netflix.iceberg:iceberg-api:0.3.2")
153156
}
154157
}
155158

‎metacat-common-server/src/main/java/com/netflix/metacat/common/server/api/v1/MetacatV1.java

+11
Original file line numberDiff line numberDiff line change
@@ -164,4 +164,15 @@ DatabaseDto getDatabase(final String catalogName, final String databaseName, fin
164164
* @return catalog
165165
*/
166166
CatalogDto getCatalog(final String catalogName);
167+
168+
/**
169+
* Get the catalog by name.
170+
*
171+
* @param catalogName catalog name
172+
* @param includeDatabaseNames if true, the response includes the database names
173+
* @param includeUserMetadata if true, the response includes the user metadata
174+
* @return catalog
175+
*/
176+
CatalogDto getCatalog(final String catalogName, final boolean includeDatabaseNames,
177+
final boolean includeUserMetadata);
167178
}

‎metacat-connector-hive/build.gradle

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ dependencies {
3131
}
3232

3333
compile("commons-dbutils:commons-dbutils")
34-
compile('com.github.Netflix.iceberg:iceberg-common:0.3.0')
35-
compile('com.github.Netflix.iceberg:iceberg-core:0.3.0')
36-
compile('com.github.Netflix.iceberg:iceberg-api:0.3.0')
34+
compile('com.github.Netflix.iceberg:iceberg-common')
35+
compile('com.github.Netflix.iceberg:iceberg-core')
36+
compile('com.github.Netflix.iceberg:iceberg-api')
3737
/*******************************
3838
* Provided Dependencies
3939
*******************************/

‎metacat-main/src/main/java/com/netflix/metacat/main/api/v1/MetacatController.java

+12-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.netflix.metacat.main.api.RequestWrapper;
3030
import com.netflix.metacat.main.services.CatalogService;
3131
import com.netflix.metacat.main.services.DatabaseService;
32+
import com.netflix.metacat.main.services.GetCatalogServiceParameters;
3233
import com.netflix.metacat.main.services.GetTableServiceParameters;
3334
import com.netflix.metacat.main.services.MViewService;
3435
import com.netflix.metacat.main.services.TableService;
@@ -495,12 +496,22 @@ public TableDto deleteMView(
495496
public CatalogDto getCatalog(
496497
@ApiParam(value = "The name of the catalog", required = true)
497498
@PathVariable("catalog-name") final String catalogName
499+
) {
500+
return getCatalog(catalogName, true, true);
501+
}
502+
503+
@Override
504+
public CatalogDto getCatalog(
505+
final String catalogName,
506+
final boolean includeUserMetadata,
507+
final boolean includeDatabaseNames
498508
) {
499509
final QualifiedName name = this.requestWrapper.qualifyName(() -> QualifiedName.ofCatalog(catalogName));
500510
return this.requestWrapper.processRequest(
501511
name,
502512
"getCatalog",
503-
() -> this.catalogService.get(name)
513+
() -> this.catalogService.get(name, GetCatalogServiceParameters.builder()
514+
.includeDatabaseNames(includeDatabaseNames).includeUserMetadata(includeUserMetadata).build())
504515
);
505516
}
506517

‎metacat-main/src/main/java/com/netflix/metacat/main/api/v1/TagController.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.netflix.metacat.main.api.RequestWrapper;
3333
import com.netflix.metacat.main.services.CatalogService;
3434
import com.netflix.metacat.main.services.DatabaseService;
35+
import com.netflix.metacat.main.services.GetCatalogServiceParameters;
3536
import com.netflix.metacat.main.services.GetTableServiceParameters;
3637
import com.netflix.metacat.main.services.MViewService;
3738
import com.netflix.metacat.main.services.TableService;
@@ -252,7 +253,8 @@ private Set<String> setResourceTags(@NonNull final TagCreateRequestDto tagCreate
252253
switch (name.getType()) {
253254
case CATALOG:
254255
//catalog service will throw exception if not found
255-
final CatalogDto catalogDto = this.catalogService.get(name);
256+
final CatalogDto catalogDto = this.catalogService.get(name, GetCatalogServiceParameters.builder()
257+
.includeDatabaseNames(false).includeUserMetadata(false).build());
256258
if (catalogDto != null) {
257259
return this.tagService.setTags(name, tags, true);
258260
}
@@ -544,7 +546,8 @@ private void removeResourceTags(final TagRemoveRequestDto tagRemoveRequestDto) {
544546
switch (name.getType()) {
545547
case CATALOG:
546548
//catalog service will throw exception if not found
547-
final CatalogDto catalogDto = this.catalogService.get(name);
549+
final CatalogDto catalogDto = this.catalogService.get(name, GetCatalogServiceParameters.builder()
550+
.includeDatabaseNames(false).includeUserMetadata(false).build());
548551
if (catalogDto != null) {
549552
this.tagService.removeTags(name, tagRemoveRequestDto.getDeleteAll(),
550553
new HashSet<>(tagRemoveRequestDto.getTags()), true);

‎metacat-main/src/main/java/com/netflix/metacat/main/services/CatalogService.java

+9
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,15 @@ public interface CatalogService {
3333
@Nonnull
3434
CatalogDto get(QualifiedName name);
3535

36+
/**
37+
* Gets the catalog. Returned dto will have details if asked.
38+
* @param name Qualified name of the catalog
39+
* @param getCatalogServiceParameters parameters
40+
* @return the information about the given catalog
41+
*/
42+
@Nonnull
43+
CatalogDto get(QualifiedName name, GetCatalogServiceParameters getCatalogServiceParameters);
44+
3645
/**
3746
* List of registered catalogs.
3847
* @return all of the registered catalogs

‎metacat-main/src/main/java/com/netflix/metacat/main/services/GetCatalogServiceParameters.java

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright 2018 Netflix, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.netflix.metacat.main.services;
18+
19+
import lombok.Builder;
20+
import lombok.Value;
21+
22+
/**
23+
* Get Catalog Parameters.
24+
*
25+
* @author amajumdar
26+
* @since 1.2.0
27+
*/
28+
@Value
29+
@Builder
30+
public class GetCatalogServiceParameters {
31+
private final boolean includeDatabaseNames;
32+
private final boolean includeUserMetadata;
33+
}

‎metacat-main/src/main/java/com/netflix/metacat/main/services/impl/CatalogServiceImpl.java

+25-10
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.netflix.metacat.main.manager.ConnectorManager;
3131
import com.netflix.metacat.main.services.CatalogService;
3232
import com.netflix.metacat.common.server.spi.MetacatCatalogConfig;
33+
import com.netflix.metacat.main.services.GetCatalogServiceParameters;
3334

3435
import javax.annotation.Nonnull;
3536
import java.util.List;
@@ -70,7 +71,7 @@ public CatalogServiceImpl(
7071
*/
7172
@Nonnull
7273
@Override
73-
public CatalogDto get(final QualifiedName name) {
74+
public CatalogDto get(final QualifiedName name, final GetCatalogServiceParameters getCatalogServiceParameters) {
7475
final Set<MetacatCatalogConfig> configs = connectorManager.getCatalogConfigs(name.getCatalogName());
7576
final CatalogDto result = new CatalogDto();
7677
result.setName(name);
@@ -83,20 +84,34 @@ public CatalogDto get(final QualifiedName name) {
8384
} else {
8485
qName = QualifiedName.ofDatabase(name.getCatalogName(), config.getSchemaWhitelist().get(0));
8586
}
86-
databases.addAll(
87-
connectorManager.getDatabaseService(qName).listNames(context, name, null, null, null)
88-
.stream().map(QualifiedName::getDatabaseName)
89-
.filter(s -> config.getSchemaBlacklist().isEmpty() || !config.getSchemaBlacklist().contains(s))
90-
.filter(s -> config.getSchemaWhitelist().isEmpty() || config.getSchemaWhitelist().contains(s))
91-
.sorted(String.CASE_INSENSITIVE_ORDER)
92-
.collect(Collectors.toList())
93-
);
87+
if (getCatalogServiceParameters.isIncludeDatabaseNames()) {
88+
databases.addAll(
89+
connectorManager.getDatabaseService(qName).listNames(context, name, null, null, null)
90+
.stream().map(QualifiedName::getDatabaseName)
91+
.filter(s -> config.getSchemaBlacklist().isEmpty() || !config.getSchemaBlacklist().contains(s))
92+
.filter(s -> config.getSchemaWhitelist().isEmpty() || config.getSchemaWhitelist().contains(s))
93+
.sorted(String.CASE_INSENSITIVE_ORDER)
94+
.collect(Collectors.toList())
95+
);
96+
}
9497
});
9598
result.setDatabases(databases);
96-
userMetadataService.populateMetadata(result, false);
99+
if (getCatalogServiceParameters.isIncludeUserMetadata()) {
100+
userMetadataService.populateMetadata(result, false);
101+
}
97102
return result;
98103
}
99104

105+
/**
106+
* {@inheritDoc}
107+
*/
108+
@Nonnull
109+
@Override
110+
public CatalogDto get(final QualifiedName name) {
111+
return get(name, GetCatalogServiceParameters.builder().includeDatabaseNames(true)
112+
.includeUserMetadata(true).build());
113+
}
114+
100115
/**
101116
* {@inheritDoc}
102117
*/

0 commit comments

Comments
 (0)