Skip to content

Commit e7ce5b1

Browse files
mnorburymnorbury
authored
chore: adding transform from class logic (#48)
Adding transform logic based on the Go `TransformWithStruct` function in the Go SDK. fixes: #3
1 parent 3fa9cbc commit e7ce5b1

8 files changed

Lines changed: 438 additions & 3 deletions

File tree

‎lib/src/main/java/io/cloudquery/schema/Column.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@
22

33
import lombok.Builder;
44
import lombok.Getter;
5+
import org.apache.arrow.vector.types.pojo.ArrowType;
56

67
@Builder
78
@Getter
89
public class Column {
910
private String name;
11+
private ArrowType type;
12+
private ColumnResolver resolver;
13+
private boolean primaryKey;
14+
private boolean ignoreInTests;
1015
}

‎lib/src/main/java/io/cloudquery/schema/Table.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package io.cloudquery.schema;
22

33
import io.cloudquery.glob.Glob;
4+
import io.cloudquery.transformers.TransformerException;
45
import lombok.Builder;
56
import lombok.Getter;
7+
import lombok.Setter;
68

79
import java.util.ArrayList;
810
import java.util.Collections;
@@ -15,6 +17,11 @@
1517
@Builder(toBuilder = true)
1618
@Getter
1719
public class Table {
20+
21+
public interface Transform {
22+
void transformTable(Table table) throws TransformerException;
23+
}
24+
1825
public static List<Table> flattenTables(List<Table> tables) {
1926
Map<String, Table> flattenMap = new HashMap<>();
2027
for (Table table : tables) {
@@ -100,12 +107,22 @@ public static int maxDepth(List<Table> tables) {
100107
}
101108

102109
private String name;
103-
110+
@Setter
104111
private Table parent;
112+
@Builder.Default
113+
private List<Column> columns = new ArrayList<>();
105114

106115
@Builder.Default
107116
private List<Table> relations = Collections.emptyList();
108117

118+
private Transform transform;
119+
120+
public void transform() throws TransformerException {
121+
if (transform != null) {
122+
transform.transformTable(this);
123+
}
124+
}
125+
109126
private Optional<Table> filterDfs(boolean parentMatched, Predicate<Table> include, Predicate<Table> exclude, boolean skipDependentTables) {
110127
if (exclude.test(this)) {
111128
return Optional.empty();
@@ -129,4 +146,12 @@ private Optional<Table> filterDfs(boolean parentMatched, Predicate<Table> includ
129146
return Optional.empty();
130147
}
131148

149+
public Optional<Column> getColumn(String name) {
150+
for (Column column : columns) {
151+
if (column.getName().equals(name)) {
152+
return Optional.of(column);
153+
}
154+
}
155+
return Optional.empty();
156+
}
132157
}

‎lib/src/main/java/io/cloudquery/transformers/Tables.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package io.cloudquery.transformers;
2+
3+
import io.cloudquery.schema.Table;
4+
5+
import java.util.List;
6+
7+
import static io.cloudquery.schema.Table.*;
8+
9+
class Tables {
10+
public static void setParents(List<Table> tables, Table parent) {
11+
for (Table table : tables) {
12+
table.setParent(parent);
13+
setParents(table.getRelations(), table);
14+
}
15+
}
16+
17+
public static void transformTables(List<Table> tables) throws TransformerException {
18+
for (Table table : tables) {
19+
table.transform();
20+
transformTables(table.getRelations());
21+
}
22+
}
23+
24+
public static void apply(List<Table> tables, List<Transform> extraTransformers) throws TransformerException {
25+
for (Table table : tables) {
26+
for (Transform extraTransformer : extraTransformers) {
27+
extraTransformer.transformTable(table);
28+
}
29+
apply(table.getRelations(), extraTransformers);
30+
}
31+
}
32+
}

‎lib/src/main/java/io/cloudquery/transformers/TransformWithClass.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package io.cloudquery.transformers;
2+
3+
import io.cloudquery.schema.Column;
4+
import io.cloudquery.schema.Column.ColumnBuilder;
5+
import io.cloudquery.schema.Table;
6+
import lombok.Builder;
7+
import lombok.Singular;
8+
9+
import java.lang.reflect.Field;
10+
import java.util.HashSet;
11+
import java.util.Set;
12+
13+
import static io.cloudquery.schema.Table.*;
14+
import static io.cloudquery.transformers.IgnoreInTestsTransformer.DefaultIgnoreInTestsTransformer;
15+
import static io.cloudquery.transformers.NameTransformer.DefaultNameTransformer;
16+
import static io.cloudquery.transformers.ResolverTransformer.DefaulResolverTransformer;
17+
import static io.cloudquery.transformers.TypeTransformer.DefaultTypeTransformer;
18+
19+
@Builder(builderMethodName = "innerBuilder")
20+
public class TransformWithClass implements Transform {
21+
@Builder.Default
22+
private NameTransformer nameTransformer = new DefaultNameTransformer();
23+
@Builder.Default
24+
private TypeTransformer typeTransformer = new DefaultTypeTransformer();
25+
@Builder.Default
26+
private ResolverTransformer resolverTransformer = new DefaulResolverTransformer();
27+
@Builder.Default
28+
private IgnoreInTestsTransformer ignoreInTestsTransformer = new DefaultIgnoreInTestsTransformer();
29+
30+
@Singular
31+
private Set<String> pkFields;
32+
private final Set<String> pkFieldsFound = new HashSet<>();
33+
34+
@Singular
35+
private Set<String> unwrapFields;
36+
37+
private final Class<?> clazz;
38+
39+
@Override
40+
public void transformTable(Table table) throws TransformerException {
41+
for (Field field : clazz.getDeclaredFields()) {
42+
if (shouldUnwrapField(field)) {
43+
for (Field innerField : field.getType().getDeclaredFields()) {
44+
addColumnFromField(table, innerField, field);
45+
}
46+
} else {
47+
addColumnFromField(table, field, null);
48+
}
49+
}
50+
51+
validatePrimaryKeysHaveBeenFound();
52+
}
53+
54+
55+
private void addColumnFromField(Table table, Field field, Field parent) throws TransformerException {
56+
String path = field.getName();
57+
String name = nameTransformer.transform(field);
58+
59+
if (parent != null) {
60+
name = nameTransformer.transform(parent) + "_" + name;
61+
path = parent.getName() + "." + path;
62+
}
63+
64+
ColumnBuilder columnBuilder = Column.builder().
65+
name(name).
66+
type(typeTransformer.transform(field)).
67+
resolver(resolverTransformer.transform(field, path)).
68+
ignoreInTests(ignoreInTestsTransformer.transform(field));
69+
70+
if (pkFields.contains(path)) {
71+
columnBuilder.primaryKey(true);
72+
pkFieldsFound.add(path);
73+
}
74+
75+
table.getColumns().add(columnBuilder.build());
76+
}
77+
78+
private boolean shouldUnwrapField(Field field) {
79+
return unwrapFields.contains(field.getName());
80+
}
81+
82+
private void validatePrimaryKeysHaveBeenFound() throws TransformerException {
83+
Set<String> missingPrimaryKeys = new HashSet<>(pkFields);
84+
missingPrimaryKeys.removeAll(pkFieldsFound);
85+
if (!missingPrimaryKeys.isEmpty()) {
86+
throw new TransformerException("failed to create all of the desired primary keys: " + missingPrimaryKeys);
87+
}
88+
}
89+
90+
public static TransformWithClassBuilder builder(Class<?> clazz) {
91+
return innerBuilder().clazz(clazz);
92+
}
93+
}

‎lib/src/main/java/io/cloudquery/transformers/TypeTransformer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ private static ArrowType transformArrowType(String name, Class<?> type) throws T
4848
if (componentType.getName().equals("byte")) {
4949
return ArrowType.Binary.INSTANCE;
5050
}
51-
return ListType.listOf(transformArrowType(name, componentType));
51+
// if element type is already json just return JSON rather than a list of JSON
52+
ArrowType elementType = transformArrowType(name, componentType);
53+
return elementType == JSONType.INSTANCE ? elementType : ListType.listOf(elementType);
5254
}
5355
if (!type.isPrimitive()) {
5456
return JSONType.INSTANCE;

‎lib/src/test/java/io/cloudquery/transformers/TablesTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package io.cloudquery.transformers;
2+
3+
import io.cloudquery.schema.Table;
4+
import io.cloudquery.schema.Table.Transform;
5+
import org.junit.jupiter.api.Test;
6+
import org.junit.jupiter.api.extension.ExtendWith;
7+
import org.mockito.Mock;
8+
import org.mockito.junit.jupiter.MockitoExtension;
9+
10+
import java.util.List;
11+
import java.util.Map;
12+
import java.util.Set;
13+
14+
import static java.util.stream.Collectors.groupingBy;
15+
import static java.util.stream.Collectors.mapping;
16+
import static java.util.stream.Collectors.toSet;
17+
import static org.junit.jupiter.api.Assertions.assertEquals;
18+
import static org.mockito.Mockito.any;
19+
import static org.mockito.Mockito.times;
20+
import static org.mockito.Mockito.verify;
21+
import static org.mockito.Mockito.verifyNoMoreInteractions;
22+
23+
@ExtendWith(MockitoExtension.class)
24+
class TablesTest {
25+
26+
@Mock
27+
private Transform child1Transformer;
28+
@Mock
29+
private Transform child1aTransformer, child1bTransformer;
30+
@Mock
31+
private Transform extraTransformer1, extraTransformer2;
32+
33+
@Test
34+
void shouldSetTheParentOnACollectionOfTables() {
35+
List<Table> tables = List.of(
36+
Table.builder().name("child1").relations(List.of(
37+
Table.builder().name("child1a").build(),
38+
Table.builder().name("child1b").build()
39+
)).build(),
40+
Table.builder().name("child2").relations(List.of(
41+
Table.builder().name("child2a").build(),
42+
Table.builder().name("child2b").build()
43+
)).build()
44+
);
45+
46+
Tables.setParents(tables, Table.builder().name("parent").build());
47+
48+
Map<String, Set<String>> tablesByParent = tablesByParent(tables);
49+
assertEquals(tablesByParent.get("parent"), Set.of("child1", "child2"));
50+
assertEquals(tablesByParent.get("child1"), Set.of("child1a", "child1b"));
51+
assertEquals(tablesByParent.get("child2"), Set.of("child2a", "child2b"));
52+
}
53+
54+
55+
@Test
56+
void shouldCallTransformOnEachTableIncludingRelations() throws TransformerException {
57+
List<Table> tables = List.of(
58+
Table.builder().name("child1").transform(child1Transformer).
59+
relations(List.of(
60+
Table.builder().name("child1a").transform(child1aTransformer).build(),
61+
Table.builder().name("child1b").transform(child1bTransformer).build()
62+
)).build()
63+
);
64+
65+
Tables.transformTables(tables);
66+
67+
verify(child1Transformer, times(1)).transformTable(any());
68+
verify(child1aTransformer, times(1)).transformTable(any());
69+
verify(child1bTransformer, times(1)).transformTable(any());
70+
}
71+
72+
@Test
73+
void shouldApplyExtraTransformationToTables() throws TransformerException {
74+
Table child1a = Table.builder().name("child1a").build();
75+
Table child1 = Table.builder().name("child1").relations(List.of(child1a)).build();
76+
77+
Tables.apply(List.of(child1), List.of(extraTransformer1, extraTransformer2));
78+
79+
verify(extraTransformer1, times(1)).transformTable(child1);
80+
verify(extraTransformer1, times(1)).transformTable(child1a);
81+
verify(extraTransformer2, times(1)).transformTable(child1);
82+
verify(extraTransformer2, times(1)).transformTable(child1a);
83+
verifyNoMoreInteractions(extraTransformer1, extraTransformer2);
84+
}
85+
86+
private static Map<String, Set<String>> tablesByParent(List<Table> tables) {
87+
return Table.flattenTables(tables).stream().
88+
collect(
89+
groupingBy(
90+
table -> table.getParent().getName(),
91+
mapping(Table::getName, toSet())
92+
)
93+
);
94+
}
95+
}

0 commit comments

Comments
 (0)