Fix DRep image base64 data URI stripping (#1966)#2138
Open
ArturWieczorek wants to merge 2 commits into
Open
Conversation
When an off-chain DRep metadata image is an inline base64 data URI (`data:<mime>;base64,...`), db-sync stored only the base64 payload in `off_chain_vote_drep_data.image_url`, stripping the `data:<mime>;base64,` prefix (and the MIME type), so the stored value was neither the source contentUrl nor a usable data URI. CIP-119 defines `contentUrl` as the full data URI for inline images, so store it verbatim. The `Image` JSON decoder now keeps any `data:`-scheme contentUrl as-is; regular URL images (which carry a sha256) are unchanged. Adds a regression test asserting a base64 data-URI contentUrl is decoded without losing its prefix.
Contributor
Author
|
Looks like this CI failure is not caused by code changes but some Nix infrastructure issue on the macOS CI runner. |
Contributor
Author
Tests Run💚 All tests passed except one that is supposed to fail for this exact change: 
[gw1] linux -- Python 3.13.11 /home/runner/work/cardano-node-tests/cardano-node-tests/run_workdir/.venv/bin/python3
self = <cardano_node_tests.tests.tests_conway.test_drep.TestDReps object at 0x7fffdd9c7d40>
cluster_manager = <cardano_node_tests.cluster_management.manager.ClusterManager object at 0x7fffd28c4cd0>
cluster = <ClusterLib: command_era=latest>
payment_addr = AddressRecord(address='addr_test1vp8c5ejp8dvnd4q5ltm6jre8vvl8q3jlyc5tpczdz5nz8ncjyl8t2', vkey_file=PosixPath('test_register_and_retire_drep[drep_metadata0-build_raw-submit_cli]_ci2_loyjkk_fund_addr_1.vkey'), skey_file=PosixPath('test_register_and_retire_drep[drep_metadata0-build_raw-submit_cli]_ci2_loyjkk_fund_addr_1.skey'))
build_method = <BuildMethods.BUILD: 'build'>, submit_method = 'cli'
drep_metadata = {'drep_metadata_file': PosixPath('/home/runner/work/cardano-node-tests/cardano-node-tests/cardano_node_tests/tests/data/drep_metadata_ipfs.json'), 'drep_metadata_url': 'https://tinyurl.com/drep-ipfs', 'expected_hash': '7b45535061dce55fca685fedfec6e41a195c4fe4ef812767c34505390b413ff3'}
@allure.link(helpers.get_vcs_link())
@submit_utils.PARAM_SUBMIT_METHOD
@common.PARAM_BUILD_METHOD_NO_EST
@pytest.mark.dbsync
@pytest.mark.dbsync_config
@pytest.mark.testnets
@pytest.mark.smoke
@pytest.mark.parametrize(
"drep_metadata",
(
{
"drep_metadata_file": DATA_DIR / "drep_metadata_url.json",
"drep_metadata_url": "https://tinyurl.com/drep-url",
"expected_hash": "18b4b10150eab04ba66c8f9cb497ff05c6c31b9c9825388481c1790ce76b6b90",
},
{
"drep_metadata_file": DATA_DIR / "drep_metadata_ipfs.json",
"drep_metadata_url": "https://tinyurl.com/drep-ipfs",
"expected_hash": "7b45535061dce55fca685fedfec6e41a195c4fe4ef812767c34505390b413ff3",
},
),
)
def test_register_and_retire_drep(
self,
cluster_manager: cluster_management.ClusterManager,
cluster: clusterlib.ClusterLib,
payment_addr: clusterlib.AddressRecord,
build_method: str,
submit_method: str,
drep_metadata: dict,
):
"""Test DRep registration and retirement.
* Register DRep
* Check that DRep was registered
* Retire DRep
* Check that DRep was retired
* Check that deposit was returned to source address
"""
temp_template = common.get_test_id(cluster)
deposit_drep_amt = cluster.g_query.get_drep_deposit()
# Make sure there's enought funds on the payment address on long running
# testnets where the DRep deposit is higher.
if deposit_drep_amt >= MAINNET_DREP_DEPOSIT:
clusterlib_utils.fund_from_faucet(
payment_addr,
cluster_obj=cluster,
all_faucets=cluster_manager.cache.addrs_data,
amount=deposit_drep_amt + 10_000_000,
)
errors_final = []
drep_metadata_file = drep_metadata["drep_metadata_file"]
drep_metadata_url = drep_metadata["drep_metadata_url"]
expected_hash = drep_metadata["expected_hash"]
# Register DRep
reqc.cli012.start(url=helpers.get_vcs_link())
drep_metadata_hash = cluster.g_governance.drep.get_metadata_hash(
drep_metadata_file=drep_metadata_file
)
with open(drep_metadata_file, encoding="utf-8") as anchor_fp:
drep_metadata_content = json.load(anchor_fp)
_url = helpers.get_vcs_link()
[r.start(url=_url) for r in (reqc.cli008, reqc.cli009, reqc.cli010, reqc.cip021)]
reg_drep = governance_utils.get_drep_reg_record(
cluster_obj=cluster,
name_template=temp_template,
drep_metadata_url=drep_metadata_url,
drep_metadata_hash=drep_metadata_hash,
)
[r.success() for r in (reqc.cli008, reqc.cli009, reqc.cli010, reqc.cip021)]
tx_files_reg = clusterlib.TxFiles(
certificate_files=[reg_drep.registration_cert],
signing_key_files=[payment_addr.skey_file, reg_drep.key_pair.skey_file],
)
tx_output_reg = clusterlib_utils.build_and_submit_tx(
cluster_obj=cluster,
name_template=f"{temp_template}_reg",
src_address=payment_addr.address,
submit_method=submit_method,
build_method=build_method,
tx_files=tx_files_reg,
deposit=reg_drep.deposit,
)
reg_out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output_reg)
assert (
clusterlib.filter_utxos(utxos=reg_out_utxos, address=payment_addr.address)[0].amount
== clusterlib.calculate_utxos_balance(tx_output_reg.txins)
- tx_output_reg.fee
- reg_drep.deposit
), f"Incorrect balance for source address `{payment_addr.address}`"
reqc.cli033.start(url=helpers.get_vcs_link())
reg_drep_state = cluster.g_query.get_drep_state(drep_vkey_file=reg_drep.key_pair.vkey_file)
assert reg_drep_state[0][0]["keyHash"] == reg_drep.drep_id, "DRep was not registered"
reqc.cli033.success()
metadata_anchor = reg_drep_state[0][1]["anchor"]
assert metadata_anchor["dataHash"] == drep_metadata_hash == expected_hash, (
"Unexpected metadata hash"
)
assert metadata_anchor["url"] == drep_metadata_url, "Unexpected metadata url"
if configuration.HAS_DBSYNC:
try:
_url = helpers.get_vcs_link()
[r.start(url=_url) for r in (reqc.db001, reqc.db006)]
drep_data = dbsync_utils.check_drep_registration(
drep=reg_drep, drep_state=reg_drep_state
)
[r.success() for r in (reqc.db001, reqc.db006)]
# The same metadata is used for all the parametrized tests. Therefore the data will
# be present in db-sync once the first parametrized test runs. Therefore it doesn't
# make sense to check the metadata for all combinations of parameters.
if build_method == clusterlib_utils.BuildMethods.BUILD and submit_method == "cli":
def _query_func():
if not drep_data:
return
dbsync_utils.check_off_chain_drep_registration(
drep_data=drep_data, metadata=drep_metadata_content
)
dbsync_utils.retry_query(query_func=_query_func, timeout=360)
except AssertionError as exc:
str_exc = str(exc)
errors_final.append(f"DB-Sync unexpected DRep registration error: {str_exc}")
reqc.cli012.success()
# Retire DRep
_url = helpers.get_vcs_link()
[r.start(url=_url) for r in (reqc.cli011, reqc.cip023)]
ret_cert = cluster.g_governance.drep.gen_retirement_cert(
cert_name=temp_template,
deposit_amt=reg_drep.deposit,
drep_vkey_file=reg_drep.key_pair.vkey_file,
)
[r.success() for r in (reqc.cli011, reqc.cip023)]
tx_files_ret = clusterlib.TxFiles(
certificate_files=[ret_cert],
signing_key_files=[payment_addr.skey_file, reg_drep.key_pair.skey_file],
)
tx_output_ret = clusterlib_utils.build_and_submit_tx(
cluster_obj=cluster,
name_template=f"{temp_template}_ret",
src_address=payment_addr.address,
submit_method=submit_method,
build_method=build_method,
tx_files=tx_files_ret,
deposit=-reg_drep.deposit,
)
reqc.cip024.start(url=helpers.get_vcs_link())
ret_drep_state = cluster.g_query.get_drep_state(drep_vkey_file=reg_drep.key_pair.vkey_file)
assert not ret_drep_state, "DRep was not retired"
reqc.cip024.success()
ret_out_utxos = cluster.g_query.get_utxo(tx_raw_output=tx_output_ret)
assert (
clusterlib.filter_utxos(utxos=ret_out_utxos, address=payment_addr.address)[0].amount
== clusterlib.calculate_utxos_balance(tx_output_ret.txins)
- tx_output_ret.fee
+ reg_drep.deposit
), f"Incorrect balance for source address `{payment_addr.address}`"
try:
dbsync_utils.check_drep_deregistration(drep=reg_drep)
except AssertionError as exc:
str_exc = str(exc)
errors_final.append(f"DB-Sync unexpected DRep deregistration error: {str_exc}")
if errors_final:
> raise AssertionError("\n".join(errors_final))
E AssertionError: DB-Sync unexpected DRep registration error: 'image_url' value is different than expected Base64 'contentUrl';
/home/runner/work/cardano-node-tests/cardano-node-tests/cardano_node_tests/tests/tests_conway/test_drep.py:502: AssertionError
------------------------------ Captured log call -------------------------------
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 3s before repeating query for the 1 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 6s before repeating query for the 2 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 11s before repeating query for the 3 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 18s before repeating query for the 4 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 27s before repeating query for the 5 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 38s before repeating query for the 6 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 51s before repeating query for the 7 time.
Tests Report 💚 |
Contributor
Author
Test Result 💚CIRun: artur/fix-drep-image-base64-data-uri-stripping Local
❯ pytest cardano_node_tests/tests/tests_conway/test_drep.py::TestDReps::test_register_and_retire_drep[drep_metadata1-build-submit_cli]
============================================================================================================== test session starts ==============================================================================================================
platform linux -- Python 3.13.3, pytest-9.0.3, pluggy-1.5.0 -- /home/artur/Projects/cardano-node-tests/.venv/bin/python
cachedir: .pytest_cache
hypothesis profile 'default'
metadata: {'Python': '3.13.3', 'Platform': 'Linux-6.8.0-124-generic-x86_64-with-glibc2.39', 'Packages': {'pytest': '9.0.3', 'pluggy': '1.5.0'}, 'Plugins': {'hypothesis': '6.148.7', 'allure-pytest': '2.15.2', 'metadata': '3.1.1', 'xdist': '3.8.0', 'order': '1.3.0', 'select': '0.1.2', 'html': '4.1.1'}, 'github actions url': '', 'cardano-node': '11.0.1', 'cardano-node rev': '390273284c76cc182cdd22b142e8a4df63edbcd3', 'cardano-node ghc': 'ghc-9.6', 'cardano-cli': '11.0.0.0', 'cardano-cli rev': '390273284c76cc182cdd22b142e8a4df63edbcd3', 'cardano-cli ghc': 'ghc-9.6', 'COMMAND_ERA': '', 'TESTNET_VARIANT': 'conway_fast', 'NUM_POOLS': '3', 'UTXO_BACKEND': '', 'MIXED_UTXO_BACKENDS': '', 'MAX_TESTS_PER_CLUSTER': 8, 'CLUSTERS_COUNT': 1, 'HAS_CC': 'True', 'cardano-node-tests rev': '275801e52aa3dd28077e40e12e1fc1de160d9f0a', 'cardano-node-tests url': 'https://github.com/IntersectMBO/cardano-node-tests/tree/275801e52aa3dd28077e40e12e1fc1de160d9f0a', 'CARDANO_NODE_SOCKET_PATH': '/var/tmp/cardonnay-of-artur/state-cluster0/bft1.socket', 'cardano-cli exe': '/nix/store/maax1flgc32fhn8cq611x5pkpi6ybnjv-cardano-cli-exe-cardano-cli-11.0.0.0/bin/cardano-cli', 'cardano-node exe': '/nix/store/yz8ks1as4z8vsiqa32gdnrfzpsdgsnyl-cardano-node-exe-cardano-node-11.0.1/bin/cardano-node', 'cardano-submit-api exe': '/nix/store/spcpa3p6kh00gj36c1ww59rgyfjnv2l2-cardano-submit-api-exe-cardano-submit-api-11.0.0/bin/cardano-submit-api', 'network magic': 42, 'HAS_DBSYNC': 'True', 'db-sync': '13.7.1.0', 'db-sync rev': '576efc987767a18ec4b3c81eccdb5307577e977b', 'db-sync ghc': 'ghc-9.6', 'db-sync exe': '/home/artur/.local/bin/cardano-db-sync', 'HAS_SMASH': 'False'}
rootdir: /home/artur/Projects/cardano-node-tests
configfile: pyproject.toml
plugins: hypothesis-6.148.7, allure-pytest-2.15.2, metadata-3.1.1, xdist-3.8.0, order-1.3.0, select-0.1.2, html-4.1.1
collected 1 item
cardano_node_tests/tests/tests_conway/test_drep.py::TestDReps::test_register_and_retire_drep[drep_metadata1-build-submit_cli]@smoke
---------------------------------------------------------------------------------------------------------------- live log setup -----------------------------------------------------------------------------------------------------------------
INFO cardano_node_tests.tests.conftest:conftest.py:225 Changed CWD to '/home/artur/Projects/cardano-node-tests/tmp/pytest-of-artur/pytest-0'.
----------------------------------------------------------------------------------------------------------------- live log call -----------------------------------------------------------------------------------------------------------------
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 3s before repeating query for the 1 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 6s before repeating query for the 2 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 11s before repeating query for the 3 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 18s before repeating query for the 4 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 27s before repeating query for the 5 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 38s before repeating query for the 6 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 51s before repeating query for the 7 time.
WARNING cardano_node_tests.utils.dbsync_utils:dbsync_utils.py:510 Sleeping 66s before repeating query for the 8 time.
PASSED [100%]
--------------------------------------------------------------------------------------------------------------- live log teardown ---------------------------------------------------------------------------------------------------------------
INFO cardano_node_tests.utils.dbsync_conn:dbsync_conn.py:35 Closing connection to db-sync database dbsync0.
========================================================================================================= 1 passed in 240.95s (0:04:00) =========================================================================================================SQL Query
PGHOST=localhost PGPORT=55432 PGUSER=postgres psql -d dbsync0 -P pager=off -c " SELECT
left(drep.image_url, 50) AS image_url_prefix,
drep.image_hash,
drep.given_name,
va.url,
va.data_hash
FROM off_chain_vote_drep_data AS drep
INNER JOIN off_chain_vote_data AS vd ON vd.id = drep.off_chain_vote_data_id
INNER JOIN voting_anchor va ON vd.voting_anchor_id = va.id
WHERE va.url = 'https://tinyurl.com/drep-ipfs';" image_url_prefix | image_hash | given_name | url | data_hash
----------------------------------------------------+------------+-----------------+-------------------------------+--------------------------------------------------------------------
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgA | | Artur Wieczorek | https://tinyurl.com/drep-ipfs | \x7b45535061dce55fca685fedfec6e41a195c4fe4ef812767c34505390b413ff3
(1 row)Full SQL QuerySELECT
drep.image_url,
drep.image_hash,
drep.given_name,
va.url,
va.data_hash
FROM off_chain_vote_drep_data AS drep
INNER JOIN off_chain_vote_data AS vd ON vd.id = drep.off_chain_vote_data_id
INNER JOIN voting_anchor va ON vd.voting_anchor_id = va.id
WHERE va.url = 'https://tinyurl.com/drep-ipfs'; |
Cmdv
approved these changes
Jun 17, 2026
Contributor
|
My concern is how this affect tools that rely on current design or the mismatch between old nad new values. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
When an off-chain DRep metadata image is an inline base64 data URI (
data:<mime>;base64,...), db-sync stored only the base64 payload inoff_chain_vote_drep_data.image_url, stripping thedata:<mime>;base64,prefix (and the MIME type), so the stored value was neither the source contentUrl nor a usable data URI.CIP-119 defines
contentUrlas the full data URI for inline images, so store it verbatim. TheImageJSON decoder now keeps anydata:-scheme contentUrl as-is; regular URL images (which carry a sha256) are unchanged.Adds a regression test asserting a base64 data-URI contentUrl is decoded without losing its prefix.
Description
Add your description here, if it fixes a particular issue please provide a link to the issue.
Checklist
fourmoluon version 0.17.0.0 (which can be run withscripts/fourmolize.sh)Migrations
If there is a breaking change, especially a big one, please add a justification here. Please elaborate
more what the migration achieves, what it cannot achieve or why a migration is not possible.