Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,17 @@ public String description() {
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
+ " instance_id: 'my-instance-id'\n"
+ " database_id: 'my-database'\n"
+ " instance: 'my-instance-id'\n"
+ " database: 'my-database'\n"
+ " query: 'SELECT * FROM table'\n"
+ "\n"
+ "It is also possible to read a table by specifying a table name and a list of columns. For "
+ "example, the following configuration will perform a read on an entire table: ::\n"
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
+ " instance_id: 'my-instance-id'\n"
+ " database_id: 'my-database'\n"
+ " instance: 'my-instance-id'\n"
+ " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ " columns: ['col1', 'col2']\n"
+ "\n"
Expand All @@ -114,8 +114,8 @@ public String description() {
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " config:\n"
+ " instance_id: 'my-instance-id'\n"
+ " database_id: 'my-database'\n"
+ " instance: 'my-instance-id'\n"
+ " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ " index: 'my-index'\n"
+ " columns: ['col1', 'col2']\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,11 @@ public String description() {
+ "\n"
+ "Example configuration for performing a write to a single table: ::\n"
+ "\n"
+ " - type: ReadFromSpanner\n"
+ " - type: WriteToSpanner\n"
+ " config:\n"
+ " project_id: 'my-project-id'\n"
+ " instance_id: 'my-instance-id'\n"
+ " database_id: 'my-database'\n"
+ " project: 'my-project-id'\n"
+ " instance: 'my-instance-id'\n"
+ " database: 'my-database'\n"
+ " table: 'my-table'\n"
+ "\n"
+ "Note: See <a href=\""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,7 @@ def _spanner_io_read_test_preprocessor(
for transform in pipeline.get('transforms', []):
if transform.get('type', '').startswith('ReadFromSpanner'):
config = transform['config']
instance, database = config['instance_id'], config['database_id']
instance, database = config['instance'], config['database']
if (table := config.get('table')) is None:
table = config.get('query', '').split('FROM')[-1].strip()
transform['type'] = 'Create'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ pipeline:

# Reading data from a Spanner database. The table used here has the following columns:
# shipment_id (String), customer_id (String), shipment_date (String), shipment_cost (Float64), customer_name (String), customer_email (String)
# ReadFromSpanner transform is called using project_id, instance_id, database_id and a query
# ReadFromSpanner transform is called using project, instance, database and a query
# A table with a list of columns can also be specified instead of a query
- type: ReadFromSpanner
name: ReadShipments
config:
project_id: 'apache-beam-testing'
instance_id: 'shipment-test'
database_id: 'shipment'
project: 'apache-beam-testing'
instance: 'shipment-test'
database: 'shipment'
query: 'SELECT * FROM shipments'

# Filtering the data based on a specific condition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ pipeline:
name: WriteSpanner
input: CreateRows
config:
project_id: 'apache-beam-testing'
instance_id: 'shipment-test'
database_id: 'shipment'
table_id: 'shipments'
project: 'apache-beam-testing'
instance: 'shipment-test'
database: 'shipment'
table: 'shipments'
error_handling:
output: my_error_output

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadOrders
config:
project_id: 'apache-beam-testing'
instance_id: 'orders-test'
database_id: 'order-database'
project: 'apache-beam-testing'
instance: 'orders-test'
database: 'order-database'
query: 'SELECT customer_id, product_id, order_date, order_amount FROM orders'

# Step 2: Enrich order details with customers details from BigQuery
Expand Down Expand Up @@ -82,10 +82,10 @@ pipeline:
name: WriteProcessedOrders
input: FilterHighValueOrders
config:
project_id: '$PROJECT'
instance_id: '$INSTANCE'
database_id: '$DATABASE'
table_id: '$TABLE'
project: '$PROJECT'
instance: '$INSTANCE'
database: '$DATABASE'
table: '$TABLE'
error_handling:
output: my_error_output

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadShipments
config:
project_id: 'apache-beam-testing'
instance_id: 'shipment-test'
database_id: 'shipment'
project: 'apache-beam-testing'
instance: 'shipment-test'
database: 'shipment'
query: 'SELECT * FROM shipments'
# Step 2: Write successful records out to GCS avro files
- type: WriteToAvro
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ pipeline:
- type: ReadFromSpanner
name: ReadShipments
config:
project_id: 'apache-beam-testing'
instance_id: 'shipment-test'
database_id: 'shipment'
project: 'apache-beam-testing'
instance: 'shipment-test'
database: 'shipment'
query: 'SELECT * FROM shipments'
# Step 2: Write records out to BigQuery
- type: WriteToBigQuery
Expand Down
Loading