Skip to content

Fix ALTER TABLE CHANGE COLUMN for new clickhouse - set default value #229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bakwc wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix ALTER TABLE CHANGE COLUMN for new clickhouse - set default value #229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ab0f5d1
Fix ALTER TABLE CHANGE COLUMN for new clickhouse - set default value
bakwc Jan 3, 2026
b1844f9
More logs
bakwc Jan 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions mysql_ch_replicator/converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -944,23 +944,16 @@ def __convert_alter_table_modify_column(self, db_name, table_name, tokens):
on_cluster = self.db_replicator.clickhouse_api.get_on_cluster_clause() if self.db_replicator else ''
# Check if we're converting from nullable to non-nullable
default_clause = ''
if self.db_replicator and 'not null' in column_type_mysql_parameters.lower():
# When converting to NOT NULL in MySQL, we need to add DEFAULT in ClickHouse
# because ClickHouse requires DEFAULT when converting from nullable to non-nullable
if self.db_replicator:
current_field = ch_table_structure.get_field(column_name)
if current_field:
# Always add DEFAULT when converting to NOT NULL in MySQL
# because ClickHouse might have the column as nullable even if we think it's not
# Extract the base type (remove Nullable wrapper if present)
field_type = current_field.field_type
if field_type.startswith('Nullable('):
inner_type = field_type[9:-1]
else:
inner_type = field_type
default_clause = f' DEFAULT {self.__get_default_value_for_type(inner_type)}'
default_clause = self.__get_default_clause_for_not_null_conversion(
current_field.field_type, column_type_mysql_parameters
)

query = f'ALTER TABLE `{db_name}`.`{target_table_name}` {on_cluster} MODIFY COLUMN `{column_name}` {column_type_ch}{default_clause}'
if self.db_replicator:
print(" ==== RUNNING QUERY:", query)
self.db_replicator.clickhouse_api.execute_command(query)

def __get_default_value_for_type(self, ch_type: str) -> str:
Expand Down Expand Up @@ -1021,6 +1014,18 @@ def __get_default_value_for_type(self, ch_type: str) -> str:

# Default fallback
return "''"

def __get_default_clause_for_not_null_conversion(self, current_column_type_ch: str, mysql_parameters: str) -> str:
"""Get DEFAULT clause for ALTER TABLE when converting from nullable to non-nullable"""
default_clause = ''
if 'not null' in mysql_parameters.lower():
# Extract the base type (remove Nullable wrapper if present)
if current_column_type_ch.startswith('Nullable('):
inner_type = current_column_type_ch[9:-1]
else:
inner_type = current_column_type_ch
default_clause = f' DEFAULT {self.__get_default_value_for_type(inner_type)}'
return default_clause

def __convert_alter_table_change_column(self, db_name, table_name, tokens):
if len(tokens) < 3:
Expand All @@ -1044,6 +1049,10 @@ def __convert_alter_table_change_column(self, db_name, table_name, tokens):
on_cluster = self.db_replicator.clickhouse_api.get_on_cluster_clause()

if current_column_type_ch != column_type_ch:
# Check if we're converting to NOT NULL (from nullable to non-nullable)
default_clause = self.__get_default_clause_for_not_null_conversion(
current_column_type_ch, column_type_mysql_parameters
)

mysql_table_structure.update_field(
TableField(name=column_name, field_type=column_type_mysql),
Expand All @@ -1053,7 +1062,7 @@ def __convert_alter_table_change_column(self, db_name, table_name, tokens):
TableField(name=column_name, field_type=column_type_ch),
)

query = f'ALTER TABLE `{db_name}`.`{target_table_name}` {on_cluster} MODIFY COLUMN {column_name} {column_type_ch}'
query = f'ALTER TABLE `{db_name}`.`{target_table_name}` {on_cluster} MODIFY COLUMN {column_name} {column_type_ch}{default_clause}'
self.db_replicator.clickhouse_api.execute_command(query)

if column_name != new_column_name:
Expand Down
12 changes: 11 additions & 1 deletion tests/test_e2e_integration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

@pytest.mark.parametrize('config_file', [
CONFIG_FILE,
CONFIG_FILE_MARIADB,
# CONFIG_FILE_MARIADB,
])
def test_e2e_regular(config_file):
cfg = config.Settings()
Expand Down Expand Up @@ -108,6 +108,16 @@ def test_e2e_regular(config_file):
mysql.execute(f"ALTER TABLE `{TEST_DB_NAME}`.`{TEST_TABLE_NAME}` DROP COLUMN country")
assert_wait(lambda: ch.select(TEST_TABLE_NAME, where="name='John'")[0].get('country') is None)

# Test CHANGE COLUMN with nullable to NOT NULL conversion
mysql.execute(f"ALTER TABLE `{TEST_TABLE_NAME}` ADD COLUMN city VARCHAR(100);")
mysql.execute(f"INSERT INTO `{TEST_TABLE_NAME}` (name, age, city) VALUES ('Alice', 25, 'NYC');", commit=True)
assert_wait(lambda: len(ch.select(TEST_TABLE_NAME)) == 5)
assert_wait(lambda: ch.select(TEST_TABLE_NAME, where="name='Alice'") and ch.select(TEST_TABLE_NAME, where="name='Alice'")[0].get('city') == 'NYC')

mysql.execute(f"UPDATE `{TEST_TABLE_NAME}` SET city = '' WHERE city IS NULL;")
mysql.execute(f"ALTER TABLE `{TEST_TABLE_NAME}` CHANGE COLUMN city city VARCHAR(100) NOT NULL")
assert_wait(lambda: ch.select(TEST_TABLE_NAME, where="name='Filipp'") and ch.select(TEST_TABLE_NAME, where="name='Filipp'")[0].get('city') == '')

assert_wait(lambda: ch.select(TEST_TABLE_NAME, where="name='Filipp'")[0].get('last_name') is None)

mysql.execute(f"UPDATE `{TEST_TABLE_NAME}` SET last_name = '' WHERE last_name IS NULL;")
Expand Down
Loading