From 322801841bfe11293782463ed67629ff94ea0346 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Tue, 2 Feb 2021 18:53:34 -0400 Subject: [PATCH 1/8] try test-failure annotation package --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d40605f77f..91b15cef3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,6 +117,10 @@ jobs: - name: Install PyYAML run: pip install PyYAML + # Install module to annotate test failures during build + - name: Install pytest-github-actions-annotate-failures + run: pip install pytest-github-actions-annotate-failures + # FIXME upgrade Docker? - name: Print kernel and Docker information From ba7b677f7c0172d769ee4e15401ce7a6d466d8ac Mon Sep 17 00:00:00 2001 From: jtotoole Date: Tue, 2 Feb 2021 22:14:43 -0400 Subject: [PATCH 2/8] add demo failing test --- apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py index 80c0792fba..ed50708aca 100644 --- a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py +++ b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py @@ -52,3 +52,6 @@ def test_get_feed_url_from_google_podcasts_url(): ) assert _get_feed_url_from_google_podcasts_url(npr_google_ep_url) == npr_feed_url + +def delete_this(): + assert 'foo' == 'bar' From e819b59a9662d39e05c870f7eb635bd8b8fc3ed9 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Tue, 2 Feb 2021 22:54:47 -0400 Subject: [PATCH 3/8] retry failed test demo --- .../tests/python/test_podcast_resolve_url.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py index ed50708aca..dc3c67a34d 100644 --- a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py +++ b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py @@ -51,7 +51,5 @@ def test_get_feed_url_from_google_podcasts_url(): 'MjA5MmZjM2ItYmMwZi00NGFiLWFlNDktM2I3YmFhMjA4ODVi?sa=X&ved=0CAUQkfYCahcKEwjg4s3umbjuAhUAAAAAHQAAAAAQAQ' ) - assert _get_feed_url_from_google_podcasts_url(npr_google_ep_url) == npr_feed_url - -def delete_this(): - assert 'foo' == 'bar' + # assert _get_feed_url_from_google_podcasts_url(npr_google_ep_url) == npr_feed_url + assert _get_feed_url_from_google_podcasts_url(npr_google_ep_url) != npr_feed_url From cd585e31e8056f3483bb338acbe4ee56d4948984 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Wed, 3 Feb 2021 12:20:03 -0400 Subject: [PATCH 4/8] try triggering annotator via env val --- .github/workflows/build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91b15cef3f..a334f7723c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,10 +117,6 @@ jobs: - name: Install PyYAML run: pip install PyYAML - # Install module to annotate test failures during build - - name: Install pytest-github-actions-annotate-failures - run: pip install pytest-github-actions-annotate-failures - # FIXME upgrade Docker? - name: Print kernel and Docker information @@ -174,6 +170,8 @@ jobs: | parallel --timeout 600 --group --joblog joblog.txt || { cat joblog.txt && exit 1; } env: + # Set so that failure annotator knows it should be run in this context + GITHUB_ACTIONS: 'true' # Map secrets to environment variables MC_BRANDWATCH_PASSWORD: ${{ secrets.MC_BRANDWATCH_PASSWORD }} MC_BRANDWATCH_USER: ${{ secrets.MC_BRANDWATCH_USER }} From 1d75019b291d8c3bdea86dbdb9e2d88ff55884f8 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Wed, 3 Feb 2021 13:09:40 -0400 Subject: [PATCH 5/8] do not delete package install --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a334f7723c..098959dd5e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,7 +117,11 @@ jobs: - name: Install PyYAML run: pip install PyYAML - # FIXME upgrade Docker? + # Install module to annotate build failures + - name: Install pytest-github-actions-annotate-failures + run: pip install pytest-github-actions-annotate-failures + + # FIXME upgrade Docker? - name: Print kernel and Docker information run: | From 3b13297dd2f785f74c2854630759501f570d5dd7 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Wed, 3 Feb 2021 14:10:44 -0400 Subject: [PATCH 6/8] change command py.test to pytest --- dev/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run_test.py b/dev/run_test.py index 99703d9f10..5e0d28277b 100755 --- a/dev/run_test.py +++ b/dev/run_test.py @@ -68,7 +68,7 @@ def docker_test_commands(all_apps_dir: str, test_file: str, verbose: bool) -> Li if test_file.endswith('.py'): test_command = [ - 'py.test', '-s', '-vv', + 'pytest', '-s', '-vv', # Disable cache because it won't be preserved '-p', 'no:cacheprovider', From adb6a463277795363ee7e7c49039a01cd565964f Mon Sep 17 00:00:00 2001 From: jtotoole Date: Wed, 3 Feb 2021 16:43:23 -0400 Subject: [PATCH 7/8] try putting plugin in single test file --- .github/workflows/build.yml | 2 - .../tests/python/test_podcast_resolve_url.py | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 098959dd5e..651514c839 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -174,8 +174,6 @@ jobs: | parallel --timeout 600 --group --joblog joblog.txt || { cat joblog.txt && exit 1; } env: - # Set so that failure annotator knows it should be run in this context - GITHUB_ACTIONS: 'true' # Map secrets to environment variables MC_BRANDWATCH_PASSWORD: ${{ secrets.MC_BRANDWATCH_PASSWORD }} MC_BRANDWATCH_USER: ${{ secrets.MC_BRANDWATCH_USER }} diff --git a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py index dc3c67a34d..b3afc11f55 100644 --- a/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py +++ b/apps/crawler-fetcher/tests/python/test_podcast_resolve_url.py @@ -1,3 +1,11 @@ +from __future__ import print_function + +import os +import sys +from collections import OrderedDict + +import pytest + # noinspection PyProtectedMember from crawler_fetcher.handlers.feed_podcast import ( _get_feed_url_from_itunes_podcasts_url, @@ -5,6 +13,74 @@ ) +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + report = outcome.get_result() + + # enable only in a workflow of GitHub Actions + # ref: https://help.github.com/en/actions/configuring-and-managing-workflows/using-environment-variables#default-environment-variables + if os.environ.get("GITHUB_ACTIONS") != "true": + return + + if report.when == "call" and report.failed: + # collect information to be annotated + filesystempath, lineno, _ = report.location + + # try to convert to absolute path in GitHub Actions + workspace = os.environ.get("GITHUB_WORKSPACE") + if workspace: + full_path = os.path.abspath(filesystempath) + try: + rel_path = os.path.relpath(full_path, workspace) + except ValueError: + # os.path.relpath() will raise ValueError on Windows + # when full_path and workspace have different mount points. + # https://github.com/utgwkk/pytest-github-actions-annotate-failures/issues/20 + rel_path = filesystempath + if not rel_path.startswith(".."): + filesystempath = rel_path + + # 0-index to 1-index + lineno += 1 + + # get the name of the current failed test, with parametrize info + longrepr = report.head_line or item.name + + # get the error message and line number from the actual error + try: + longrepr += "\n\n" + report.longrepr.reprcrash.message + lineno = report.longrepr.reprcrash.lineno + + except AttributeError: + pass + + print( + _error_workflow_command(filesystempath, lineno, longrepr), file=sys.stderr + ) + + +def _error_workflow_command(filesystempath, lineno, longrepr): + # Build collection of arguments. Ordering is strict for easy testing + details_dict = OrderedDict() + details_dict["file"] = filesystempath + if lineno is not None: + details_dict["line"] = lineno + + details = ",".join("{}={}".format(k, v) for k, v in details_dict.items()) + + if longrepr is None: + return "\n::error {}".format(details) + else: + longrepr = _escape(longrepr) + return "\n::error {}::{}".format(details, longrepr) + + +def _escape(s): + return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + def test_get_feed_url_from_itunes_podcasts_url(): # noinspection PyTypeChecker assert _get_feed_url_from_itunes_podcasts_url(None) is None From 95edada4b20e53e7f0a2045b4fa4ebd1efb9a9c1 Mon Sep 17 00:00:00 2001 From: jtotoole Date: Thu, 4 Feb 2021 13:39:14 -0400 Subject: [PATCH 8/8] set GITHUB_ACTIONS flag in run_test.py --- dev/run_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/run_test.py b/dev/run_test.py index 5e0d28277b..5702ed245b 100755 --- a/dev/run_test.py +++ b/dev/run_test.py @@ -73,6 +73,9 @@ def docker_test_commands(all_apps_dir: str, test_file: str, verbose: bool) -> Li # Disable cache because it won't be preserved '-p', 'no:cacheprovider', + # flag needed for running test annotation plugin in GH build (of no consequence if running locally) + '-e', 'GITHUB_ACTIONS=true', + test_path_in_container, ] elif test_file.endswith('.t'):