X Tutup
Skip to content

Commit b00ea09

Browse files
billyjacobsonbillyjacobson
authored
Docs: Bigtable + Dataflow hello world cleanups (GoogleCloudPlatform#6314)
* Docs: Bigtable + Dataflow hello world cleanups * Fix tests * Remove guava dep * Fix dependecy issue and add a test with explanation of the issue * Replying to comments
1 parent 110e8b0 commit b00ea09

File tree

5 files changed

+84
-14
lines changed

5 files changed

+84
-14
lines changed

‎bigtable/beam/helloworld/README.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Bigtable Dataflow Hello World
2+
3+
This folder has samples showing how to read and write data with Bigtable in a
4+
Dataflow pipeline.
5+
6+
## Running instructions
7+
8+
1. Create a Bigtable instance
9+
10+
1. Create a table with a column family stats_summary
11+
12+
```cbt createtable mobile-time-series families="stats_summary"```
13+
14+
1. Set up the environment variables
15+
16+
```
17+
GOOGLE_CLOUD_PROJECT=your-project-id
18+
BIGTABLE_PROJECT=your-project-id
19+
INSTANCE_ID=your-instance-id
20+
TABLE_ID=your-table-id
21+
REGION=us-central1
22+
```
23+
24+
1. Run the command to write the data
25+
26+
```
27+
mvn compile exec:java -Dexec.mainClass=HelloWorldWrite \
28+
"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT --bigtableProjectId=$BIGTABLE_PROJECT \
29+
--bigtableInstanceId=$INSTANCE_ID --bigtableTableId=$TABLE_ID \
30+
--runner=dataflow --region=$REGION"
31+
```
32+
33+
1. Run the command to read the data
34+
35+
```
36+
mvn compile exec:java -Dexec.mainClass=HelloWorldRead \
37+
"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT --bigtableProjectId=$BIGTABLE_PROJECT \
38+
--bigtableInstanceId=$INSTANCE_ID --bigtableTableId=$TABLE_ID \
39+
--runner=dataflow --region=$REGION"
40+
```
41+
42+
43+
### Running locally
44+
45+
To run either of these programs on your local machine, you can use the direct dataflow runner
46+
by setting `--runner=direct`. If you're running it locally, you don't need the
47+
`--project` or `--region` parameters either.

‎bigtable/beam/helloworld/pom.xml‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
limitations under the License.
1616
-->
1717
<project xmlns="http://maven.apache.org/POM/4.0.0"
18-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
2020
<modelVersion>4.0.0</modelVersion>
2121

2222
<groupId>com.example.bigtable</groupId>
@@ -38,6 +38,7 @@
3838
<maven.compiler.source>1.8</maven.compiler.source>
3939
<maven.compiler.target>1.8</maven.compiler.target>
4040
<apache_beam.version>2.31.0</apache_beam.version>
41+
<exec.cleanupDaemonThreads>false</exec.cleanupDaemonThreads>
4142
</properties>
4243

4344
<dependencies>
@@ -53,7 +54,8 @@
5354
</dependency>
5455
<dependency>
5556
<groupId>org.apache.beam</groupId>
56-
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
57+
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core
58+
</artifactId>
5759
<version>${apache_beam.version}</version>
5860
</dependency>
5961

@@ -76,6 +78,18 @@
7678
<artifactId>truth</artifactId>
7779
<version>1.1.3</version>
7880
<scope>test</scope>
81+
<exclusions>
82+
<exclusion>
83+
<groupId>com.google.guava</groupId>
84+
<artifactId>guava</artifactId>
85+
</exclusion>
86+
</exclusions>
87+
</dependency>
88+
<!-- manually declare guava to override version from shared-configuration -->
89+
<dependency>
90+
<groupId>com.google.guava</groupId>
91+
<artifactId>guava</artifactId>
92+
<version>31.0.1-jre</version>
7993
</dependency>
8094
</dependencies>
8195
</project>

‎bigtable/beam/helloworld/src/main/java/HelloWorldRead.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ public interface BigtableOptions extends DataflowPipelineOptions {
7676
void setBigtableInstanceId(String bigtableInstanceId);
7777

7878
@Description("The Bigtable table ID in the instance.")
79-
@Default.String("bigtable-table")
79+
@Default.String("mobile-time-series")
8080
String getBigtableTableId();
8181

8282
void setBigtableTableId(String bigtableTableId);

‎bigtable/beam/helloworld/src/main/java/HelloWorldWrite.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public interface BigtableOptions extends DataflowPipelineOptions {
8484
void setBigtableInstanceId(String bigtableInstanceId);
8585

8686
@Description("The Bigtable table ID in the instance.")
87-
@Default.String("bigtable-table")
87+
@Default.String("mobile-time-series")
8888
String getBigtableTableId();
8989

9090
void setBigtableTableId(String bigtableTableId);

‎bigtable/beam/helloworld/src/test/java/HelloWorldTest.java‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,12 @@
3939
import org.junit.Test;
4040

4141
public class HelloWorldTest {
42+
4243
private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE";
4344
private static final String TABLE_ID =
4445
"mobile-time-series-" + UUID.randomUUID().toString().substring(0, 20);
4546
private static final String COLUMN_FAMILY_NAME = "stats_summary";
47+
private static final String REGION = "us-central1";
4648

4749
private static String projectId;
4850
private static String instanceId;
@@ -98,13 +100,20 @@ public static void afterClass() {
98100
}
99101
}
100102

103+
/**
104+
* This is tested via dataflow to ensure there are no dependency conflicts which can cause
105+
* runtime errors. To run this test faster locally, comment out the runner parameter.
106+
*/
101107
@Test
102108
public void testWrite() {
103109
HelloWorldWrite.main(
104-
new String[] {
105-
"--bigtableProjectId=" + projectId,
106-
"--bigtableInstanceId=" + instanceId,
107-
"--bigtableTableId=" + TABLE_ID
110+
new String[]{
111+
"--bigtableProjectId=" + projectId,
112+
"--project=" + projectId,
113+
"--bigtableInstanceId=" + instanceId,
114+
"--bigtableTableId=" + TABLE_ID,
115+
"--region=" + REGION,
116+
"--runner=dataflow",
108117
});
109118

110119
long count = 0;
@@ -121,16 +130,16 @@ public void testWrite() {
121130
System.out.println(
122131
"Unable to initialize service client, as a network error occurred: \n" + e.toString());
123132
}
124-
assertThat(count).isGreaterThan(0);
133+
assertThat(count).isEqualTo(3);
125134
}
126135

127136
@Test
128137
public void testRead() {
129138
HelloWorldRead.main(
130-
new String[] {
131-
"--bigtableProjectId=" + projectId,
132-
"--bigtableInstanceId=" + instanceId,
133-
"--bigtableTableId=" + TABLE_ID
139+
new String[]{
140+
"--bigtableProjectId=" + projectId,
141+
"--bigtableInstanceId=" + instanceId,
142+
"--bigtableTableId=" + TABLE_ID
134143
});
135144

136145
String output = bout.toString();

0 commit comments

Comments
 (0)
X Tutup