diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..765b447 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,280 @@ +# EditorConfig: https://editorconfig.org +# Applied by Visual Studio, VS Code, Rider, and `dotnet format`. + +root = true + +# ----------------------------- +# All files +# ----------------------------- +[*] +charset = utf-8 +end_of_line = crlf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +# ----------------------------- +# Markdown +# ----------------------------- +[*.md] +trim_trailing_whitespace = false + +# ----------------------------- +# YAML / GitHub Actions +# ----------------------------- +[*.{yml,yaml}] +indent_size = 2 + +# ----------------------------- +# JSON +# ----------------------------- +[*.json] +indent_size = 2 + +# ----------------------------- +# XML / MSBuild project files +# ----------------------------- +[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}] +indent_style = tab +indent_size = 4 + +[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}] +indent_style = tab +indent_size = 4 + +[*.{sln,DotSettings}] +indent_style = tab +indent_size = 4 + +# ----------------------------- +# Shell / scripts +# ----------------------------- +[*.{sh,ps1,cmd,bat}] +end_of_line = lf +indent_size = 2 + +[*.{cmd,bat,ps1}] +end_of_line = crlf + +# ----------------------------- +# C# source +# ----------------------------- +[*.cs] +indent_style = space +indent_size = 4 + +#### .NET coding conventions #### +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +# this. / Me. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members:suggestion +dotnet_style_readonly_field = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +dotnet_style_prefer_compound_assignment = true:suggestion +dotnet_style_prefer_simplified_interpolation = true:suggestion +dotnet_style_prefer_simplified_boolean_expressions = true:suggestion +dotnet_style_namespace_match_folder = true:suggestion + +# Null checking +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_null_check_over_type_check = true:suggestion + +# Parameter preferences +dotnet_code_quality_unused_parameters = all:suggestion + +# Suppression +dotnet_remove_unnecessary_suppression_exclusions = none + +#### C# coding conventions #### +# var preferences — prefer explicit types (matches existing sample style) +csharp_style_var_for_built_in_types = false:suggestion +csharp_style_var_when_type_is_apparent = false:suggestion +csharp_style_var_elsewhere = false:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = false:silent +csharp_style_expression_bodied_constructors = false:silent +csharp_style_expression_bodied_operators = false:silent +csharp_style_expression_bodied_properties = true:silent +csharp_style_expression_bodied_indexers = true:silent +csharp_style_expression_bodied_accessors = true:silent +csharp_style_expression_bodied_lambdas = true:silent +csharp_style_expression_bodied_local_functions = false:silent + +# Pattern matching +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion +csharp_style_prefer_pattern_matching = true:silent +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion + +# Null checking +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Modifier preferences +csharp_prefer_static_local_function = true:suggestion +csharp_preferred_modifier_order = public,private,protected,internal,file,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,required,volatile,async:silent +csharp_style_prefer_readonly_struct = true:suggestion +csharp_style_prefer_readonly_struct_member = true:suggestion + +# Code-block preferences (Allman / new-line braces — matches this repo) +csharp_prefer_braces = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_namespace_declarations = block_scoped:suggestion +csharp_style_prefer_method_group_conversion = true:silent +csharp_style_prefer_top_level_statements = false:suggestion +csharp_style_prefer_primary_constructors = false:suggestion + +# Expression-level preferences +csharp_prefer_simple_default_expression = true:suggestion +csharp_style_deconstructed_variable_declaration = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_prefer_utf8_string_literals = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion + +# 'using' directive preferences +csharp_using_directive_placement = outside_namespace:suggestion + +# whether to prefer new() without type when type is apparent +dotnet_style_prefer_collection_expression = when_types_exactly_match:suggestion + +#### C# formatting rules #### +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_case_contents = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents_when_block = true + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +#### Naming conventions #### +dotnet_naming_style.pascal_case_style.capitalization = pascal_case +dotnet_naming_style.camel_case_style.capitalization = camel_case +dotnet_naming_style.i_prefix_style.required_prefix = I +dotnet_naming_style.i_prefix_style.capitalization = pascal_case +dotnet_naming_style.underscore_camel_case.required_prefix = _ +dotnet_naming_style.underscore_camel_case.capitalization = camel_case + +# Interfaces: IPascalCase +dotnet_naming_rule.interface_should_be_begins_with_i.severity = true +dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface_symbols +dotnet_naming_rule.interface_should_be_begins_with_i.style = i_prefix_style +dotnet_naming_symbols.interface_symbols.applicable_kinds = interface +dotnet_naming_symbols.interface_symbols.applicable_accessibilities = * +dotnet_naming_symbols.interface_symbols.required_modifiers = + +# Types / non-field members: PascalCase +dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.types_should_be_pascal_case.symbols = types_and_namespaces +dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.types_and_namespaces.applicable_kinds = class,struct,interface,enum,delegate,namespace +dotnet_naming_symbols.types_and_namespaces.applicable_accessibilities = * + +dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members +dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.non_field_members.applicable_kinds = property,event,method +dotnet_naming_symbols.non_field_members.applicable_accessibilities = * + +# Constants: PascalCase +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Private / internal instance fields: _camelCase +dotnet_naming_rule.private_fields_should_be_underscore_camel.severity = suggestion +dotnet_naming_rule.private_fields_should_be_underscore_camel.symbols = private_fields +dotnet_naming_rule.private_fields_should_be_underscore_camel.style = underscore_camel_case +dotnet_naming_symbols.private_fields.applicable_kinds = field +dotnet_naming_symbols.private_fields.applicable_accessibilities = private,private_protected,protected,internal,protected_internal +dotnet_naming_symbols.private_fields.required_modifiers = + +# Static fields (non-const): PascalCase +dotnet_naming_rule.static_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.static_fields_should_be_pascal_case.symbols = static_fields +dotnet_naming_rule.static_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.applicable_accessibilities = * +dotnet_naming_symbols.static_fields.required_modifiers = static + +# Local variables / parameters: camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter,local +dotnet_naming_symbols.locals_and_parameters.applicable_accessibilities = * diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..faf8844 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default owners for code review requests +* @VAllens diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..2f9f798 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,48 @@ +name: Bug report +description: Report a problem with the crawler sample +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for filing a bug. Please include enough detail to reproduce the issue. + - type: textarea + id: description + attributes: + label: Description + description: What happened? + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Exact commands / actions + value: | + 1. + 2. + 3. + validations: + required: true + - type: input + id: sdk + attributes: + label: .NET SDK version + placeholder: e.g. 10.0.302 + validations: + required: true + - type: dropdown + id: os + attributes: + label: OS + options: + - Windows + - Linux + - macOS + validations: + required: true + - type: textarea + id: logs + attributes: + label: Logs / exception + render: shell diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..63afa00 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,23 @@ +name: Feature request +description: Suggest an improvement to the sample +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: Problem / motivation + description: What problem would this solve for learners or maintainers? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed change + description: Describe the API / docs / sample change you want + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Optional diff --git a/.github/codex/prompts/review.md b/.github/codex/prompts/review.md new file mode 100644 index 0000000..ef0abf6 --- /dev/null +++ b/.github/codex/prompts/review.md @@ -0,0 +1,12 @@ +# Codex PR review rubric + +When reviewing a pull request in this repository, focus on: + +- Correctness bugs and exception paths +- Browser / page resource leaks +- Fragile HTML selectors and scraping breakage risks +- Security issues (secrets, unsafe deserialization, SSRF-like fetches) +- Missing docs / CI updates when runtime or packages change + +Be concise and specific. Prefer actionable findings with file paths. +Keep feedback proportional to the size of the PR. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9740970 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: nuget + directory: /src/CrawlerSamples.ConsoleApp + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ba14dde --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "10.0.x" + + - name: Restore + run: dotnet restore CrawlerSamples.sln --use-current-runtime + + - name: Build + run: dotnet build CrawlerSamples.sln -c Release --no-restore + + - name: Publish + run: > + dotnet publish src/CrawlerSamples.ConsoleApp/CrawlerSamples.ConsoleApp.csproj + -c Release + --use-current-runtime + --self-contained + -o artifacts/publish + + - name: Upload publish artifact + uses: actions/upload-artifact@v7 + with: + name: crawler-samples-${{ matrix.os }} + path: artifacts/publish + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/codex-review.yml b/.github/workflows/codex-review.yml new file mode 100644 index 0000000..18d7a22 --- /dev/null +++ b/.github/workflows/codex-review.yml @@ -0,0 +1,81 @@ +# Optional Codex PR review workflow. +# Enable by setting repository variable CODEX_REVIEW_ENABLED=true +# and adding repository secret OPENAI_API_KEY. +# +# This aligns with Codex for Open Source API-credit usage for +# pull request review / maintainer automation. +name: Codex PR Review + +on: + pull_request: + types: [opened, synchronize, reopened] + +concurrency: + group: codex-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + codex: + if: ${{ vars.CODEX_REVIEW_ENABLED == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + final_message: ${{ steps.run_codex.outputs.final-message }} + steps: + - name: Checkout PR merge commit + uses: actions/checkout@v7 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + persist-credentials: false + + - name: Pre-fetch base and head refs + env: + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + git fetch --no-tags origin \ + "$PR_BASE_REF" \ + "+refs/pull/$PR_NUMBER/head" + + - name: Run Codex + id: run_codex + uses: openai/codex-action@v1 + with: + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + permission-profile: ":workspace" + prompt: | + Follow the review guidelines in .github/codex/prompts/review.md + + This is PR #${{ github.event.pull_request.number }} for ${{ github.repository }}. + + Review ONLY the changes introduced by the PR. Consider: + git log --oneline ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} + git diff --stat ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} + + Pull request title and body: + ---- + ${{ github.event.pull_request.title }} + ${{ github.event.pull_request.body }} + + post_feedback: + if: ${{ vars.CODEX_REVIEW_ENABLED == 'true' && needs.codex.outputs.final_message != '' }} + needs: codex + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Report Codex feedback + uses: actions/github-script@v9 + env: + CODEX_FINAL_MESSAGE: ${{ needs.codex.outputs.final_message }} + with: + github-token: ${{ github.token }} + script: | + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + body: process.env.CODEX_FINAL_MESSAGE, + }); diff --git a/.gitignore b/.gitignore index 3c4efe2..d5a18de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,68 +1,125 @@ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore # User-specific files +*.rsuser *.suo *.user *.userosscache *.sln.docstates +*.env # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs +# Mono auto generated files +mono_crash.* + # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ -x64/ -x86/ + +[Dd]ebug/x64/ +[Dd]ebugPublic/x64/ +[Rr]elease/x64/ +[Rr]eleases/x64/ +bin/x64/ +obj/x64/ + +[Dd]ebug/x86/ +[Dd]ebugPublic/x86/ +[Rr]elease/x86/ +[Rr]eleases/x86/ +bin/x86/ +obj/x86/ + +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +[Aa][Rr][Mm]64[Ee][Cc]/ bld/ -[Bb]in/ [Oo]bj/ +[Oo]ut/ [Ll]og/ +[Ll]ogs/ + +# Build results on 'Bin' directories +**/[Bb]in/* +# Uncomment if you have tasks that rely on *.refresh files to move binaries +# (https://github.com/github/gitignore/pull/3736) +#!**/[Bb]in/*.refresh -# Visual Studio 2015 cache/options directory +# Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ +# Visual Studio 2017 auto generated files +Generated\ Files/ + # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* +*.trx -# NUNIT +# NUnit *.VisualState.xml TestResult.xml +nunit-*.xml + +# Approval Tests result files +*.received.* # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c -# DNX +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core project.lock.json project.fragment.lock.json artifacts/ +.artifacts/ +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio *_i.c *_p.c -*_i.h +*_h.h *.ilk *.meta *.obj +*.idb +*.iobj *.pch *.pdb +*.ipdb *.pgc *.pgd *.rsp +# but not Directory.Build.rsp, as it configures directory-level build defaults +!Directory.Build.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj +*_wpftmp.csproj *.log +*.tlog *.vspscc *.vssscc .builds @@ -90,6 +147,9 @@ ipch/ *.vspx *.sap +# Visual Studio Trace Files +*.e2e + # TFS 2012 Local Workspace $tf/ @@ -101,17 +161,28 @@ _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user -# JustCode is a .NET coding add-in -.JustCode - # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + # NCrunch _NCrunch_* +.NCrunch_* .*crunch*.local.xml nCrunchTemp_* @@ -141,9 +212,9 @@ publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml -# TODO: Comment the next line if you want to checkin your web deploy settings +# Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted -#*.pubxml +*.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to @@ -153,13 +224,15 @@ PublishScripts/ # NuGet Packages *.nupkg +# NuGet Symbol Packages +*.snupkg # The packages folder can be ignored because of Package Restore -**/packages/* +**/[Pp]ackages/* # except build/, which is used as an MSBuild target. -!**/packages/build/ +!**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed -#!**/packages/repositories.config -# NuGet v3's project.json files produces more ignoreable files +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets @@ -176,12 +249,15 @@ AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt +*.appx +*.appxbundle +*.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache -!*.[Cc]ache/ +!?*.[Cc]ache/ # Others ClientBin/ @@ -192,9 +268,12 @@ ClientBin/ *.jfm *.pfx *.publishsettings -node_modules/ orleans.codegen.cs +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ @@ -209,15 +288,22 @@ _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak # SQL Server files *.mdf *.ldf +*.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ @@ -227,6 +313,7 @@ FakesAssemblies/ # Node.js Tools for Visual Studio .ntvs_analysis.dat +node_modules/ # Visual Studio 6 build log *.plg @@ -234,6 +321,17 @@ FakesAssemblies/ # Visual Studio 6 workspace options file *.opt +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts @@ -243,19 +341,89 @@ FakesAssemblies/ _Pvt_Extensions # Paket dependency manager -.paket/paket.exe +**/.paket/paket.exe paket-files/ # FAKE - F# Make -.fake/ +**/.fake/ -# JetBrains Rider -.idea/ -*.sln.iml - -# CodeRush -.cr/ +# CodeRush personal settings +**/.cr/personal # Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc \ No newline at end of file +**/__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +#tools/** +#!tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog +MSBuild_Logs/ + +# AWS SAM Build and Temporary Artifacts folder +.aws-sam + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +**/.mfractor/ + +# Local History for Visual Studio +**/.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +**/.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..7dedc46 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,22 @@ +# Repository guidance for AI coding agents + +This repository is a .NET sample demonstrating Puppeteer Sharp + AngleSharp crawling. + +## Goals + +- Keep the sample small, readable, and runnable with `dotnet run` +- Prefer correctness and resource safety (browser/page disposal) over clever abstractions +- Preserve MIT licensing and public-sample intent + +## When changing code + +1. Run `dotnet build CrawlerSamples.sln -c Release` after edits +2. Do not add secrets, credentials, or private scrape targets +3. If HTML selectors change, update `CreateModelWithAngleSharp` and mention it in the PR +4. Keep README/CI docs in sync with target framework and package versions +5. Avoid expanding scope into a full crawler framework unless explicitly requested + +## CI expectations + +- GitHub Actions builds on push/PR to `main` +- Optional Codex PR review workflow runs only when `CODEX_REVIEW_ENABLED=true` diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f9d6cf7 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,66 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement via the maintainer +contact details on [GitHub](https://github.com/VAllens). + +All complaints will be reviewed and investigated promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0f7eecf --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,48 @@ +# Contributing to Crawler Samples + +Thanks for helping keep this sample useful for the .NET crawler / scraping community. + +## Development setup + +1. Install the [.NET 10 SDK](https://dotnet.microsoft.com/download) +2. Clone the repository +3. Restore and build: + +```bash +dotnet restore +dotnet build CrawlerSamples.sln -c Release +``` + +4. Run the sample: + +```bash +dotnet run --project src/CrawlerSamples.ConsoleApp +``` + +## Pull requests + +- Keep changes focused and documented in the PR description +- Prefer small PRs that are easy to review +- Update README / comments when selectors, target pages, or runtime requirements change +- Make sure `dotnet build CrawlerSamples.sln -c Release` succeeds locally +- Avoid committing secrets, local Chromium caches, or build outputs + +## Coding guidelines + +- Target `net10.0` and keep nullable reference types enabled +- Prefer explicit resource cleanup for browser / page objects +- Keep HTML selectors as stable as possible; document fragile CSS-module selectors +- Do not scrape private or authenticated content in this sample + +## Issues + +When filing a bug, include: + +- OS and .NET SDK version +- Package versions (`PuppeteerSharp`, `AngleSharp`) +- Exact exception / stack trace +- Whether Chromium downloaded successfully + +## Code of conduct + +This project follows the [Contributor Covenant](CODE_OF_CONDUCT.md). diff --git a/LICENSE b/LICENSE index b48df30..8345c25 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Darío Kondratiuk +Copyright (c) 2017-2026 Allen (VAllens) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/README.md b/README.md index 923db67..fab405c 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,100 @@ # Crawler Samples -This is a `PuppeteerSharp` + `AngleSharp` crawler console app samples, used `C# 7.1` coding and dotnet core build. -# Useful links -* [Puppeteer Sharp](https://github.com/kblok/puppeteer-sharp) -* [AngleSharp](https://github.com/AngleSharp/AngleSharp) +[![CI](https://github.com/VAllens/CrawlerSamples/actions/workflows/ci.yml/badge.svg)](https://github.com/VAllens/CrawlerSamples/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) +[![.NET](https://img.shields.io/badge/.NET-10.0-512BD4)](https://dotnet.microsoft.com/) +A small, practical **C#** sample that crawls a modern JavaScript-rendered page with [Puppeteer Sharp](https://github.com/hardkoded/puppeteer-sharp), then parses and extracts structured data with [AngleSharp](https://github.com/AngleSharp/AngleSharp). -# Usage -Open this sample project in the `Visual Studio 2017` and press **`F5`** key run it. -### Or -Open this sample project in the `Visual Studio 2017` and enter **`Build(B) --> Publish(H)`** in the top toolbar. +The demo targets the public [.NET organization repositories](https://github.com/orgs/dotnet/repositories) page on GitHub and prints the extracted repository list as JSON. -Go to `build\win\netcoreapp2.0\x64` or `build\win\netcoreapp2.0\x86` folder, run `CrawlerSamples.ConsoleApp.exe` here. +## Why this sample exists +Many real-world pages are no longer static HTML. This repository shows a maintainable pattern for: -# Result picture -Sample Snapshoot +1. Rendering dynamic pages in a headless Chromium browser (`PuppeteerSharp`) +2. Parsing the resulting DOM with a standards-based HTML parser (`AngleSharp`) +3. Mapping DOM nodes into typed models and serializing them with `System.Text.Json` -# Note -For the first time, it will download the `download-Win64-536395.zip` portable installation package from the network to the .local-chromium directory of the current program. +It is intended as a learning / reference project for .NET developers building crawlers, scrapers, or HTML extraction tools. -It will some time to wait here. +## Requirements -If a network problem occurs during downloading, which results in download failure, it throws an exception and exits the program. +- [.NET 10 SDK](https://dotnet.microsoft.com/download) +- Network access on first run (downloads a Chromium revision used by Puppeteer Sharp) +## Quick start -# Author -Author: [Allen](http://vallen.cnblogs.com) +```bash +dotnet restore +dotnet run --project src/CrawlerSamples.ConsoleApp +``` -# License -MIT +Or open `CrawlerSamples.sln` in Visual Studio / VS Code / Rider and run the console app. + +### Publish (optional) + +```bash +dotnet publish src/CrawlerSamples.ConsoleApp -c Release -r win-x64 --self-contained true -o build/win/net10.0/x64 +``` + +You can also use the included publish profiles under `src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/`. + +## Sample output + +The app prints a JSON array of repository models, for example: + +```json +[ + { + "url": "/dotnet/runtime", + "visibility": "Public", + "title": "runtime", + "description": ".NET is a cross-platform runtime...", + "language": "C#", + "license": "MIT" + } +] +``` + +Screenshot: + +Sample snapshot + +## Project layout + +```text +CrawlerSamples.sln +src/CrawlerSamples.ConsoleApp/ + Program.cs # Crawl + parse flow + RepoModel.cs # Extracted model + JSON source generation +``` + +## Notes + +- On first run, Puppeteer Sharp downloads a Chromium package into a local cache. This can take a while depending on your network. +- If the Chromium download fails, the process exits with an exception. +- GitHub's UI markup can change. If selectors stop matching, update them in `CreateModelWithAngleSharp`. +- The sample waits for a key press only in interactive terminals; redirected / CI environments exit immediately after printing results. + +## Useful links + +- [Puppeteer Sharp](https://github.com/hardkoded/puppeteer-sharp) +- [AngleSharp](https://github.com/AngleSharp/AngleSharp) +- [Codex for Open Source](https://developers.openai.com/community/codex-for-oss) + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md). Please open issues and pull requests — CI builds on every PR. + +## Security + +See [SECURITY.md](SECURITY.md). + +## Author + +[Allen (VAllens)](https://github.com/VAllens) + +## License + +[MIT](LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..7cbe720 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,29 @@ +# Security Policy + +## Supported versions + +This is a sample / educational repository. Security fixes are applied on the latest `main` branch only. + +## Reporting a vulnerability + +Please **do not** open a public issue for security-sensitive reports. + +Instead, email the maintainer via the contact details on the [GitHub profile](https://github.com/VAllens), or open a private [GitHub Security Advisory](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) if available for this repository. + +Include: + +- A short description of the issue +- Steps to reproduce +- Impact assessment (if known) +- Any suggested fix + +We aim to acknowledge reports within 7 days. + +## Scope notes + +This sample launches a headless browser and fetches public web content. When adapting it for production: + +- Respect target site `robots.txt`, terms of service, and rate limits +- Do not store or log credentials in source control +- Pin package versions and keep dependencies updated (Dependabot is enabled) +- Treat HTML from the network as untrusted input diff --git a/SampleSnapshoot.png b/SampleSnapshoot.png index 73e5978..20cb160 100644 Binary files a/SampleSnapshoot.png and b/SampleSnapshoot.png differ diff --git a/build/win/.gitignore b/build/win/.gitignore deleted file mode 100644 index 3ad085d..0000000 --- a/build/win/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Ignore everything in this directory -* -# Except this file !.gitignore \ No newline at end of file diff --git a/nuget.config b/nuget.config new file mode 100644 index 0000000..4d736c1 --- /dev/null +++ b/nuget.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/CrawlerSamples.ConsoleApp/CarModel.cs b/src/CrawlerSamples.ConsoleApp/CarModel.cs deleted file mode 100644 index 07f8549..0000000 --- a/src/CrawlerSamples.ConsoleApp/CarModel.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace CrawlerSamples -{ - public class CarModel - { - public virtual string Title { get; set; } - - public virtual string ImageUrl { get; set; } - - public virtual string ProductUrl { get; set; } - - public virtual string OriginalPrice { get; set; } - - public virtual string PresentPrice { get; set; } - - public virtual string OrdersNumber { get; set; } - - public virtual string Tip { get; set; } - } -} \ No newline at end of file diff --git a/src/CrawlerSamples.ConsoleApp/CrawlerSamples.ConsoleApp.csproj b/src/CrawlerSamples.ConsoleApp/CrawlerSamples.ConsoleApp.csproj index c85661b..07ebc46 100644 --- a/src/CrawlerSamples.ConsoleApp/CrawlerSamples.ConsoleApp.csproj +++ b/src/CrawlerSamples.ConsoleApp/CrawlerSamples.ConsoleApp.csproj @@ -1,16 +1,18 @@  - - Exe - netcoreapp3.0 - 7.1 - CrawlerSamples - + + Exe + net10.0 + CrawlerSamples + enable + disable + latest + true + - - - - - + + + + diff --git a/src/CrawlerSamples.ConsoleApp/Program.cs b/src/CrawlerSamples.ConsoleApp/Program.cs index e5c46e5..88ef2fb 100644 --- a/src/CrawlerSamples.ConsoleApp/Program.cs +++ b/src/CrawlerSamples.ConsoleApp/Program.cs @@ -1,113 +1,155 @@ -/* +/* * This is a Puppeteer+AngleSharp crawler console app samples */ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; using AngleSharp; using AngleSharp.Dom; +using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; -using Newtonsoft.Json; using PuppeteerSharp; +using PuppeteerSharp.BrowserData; +using System; +using System.Linq; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Threading.Tasks; namespace CrawlerSamples { - internal class Program + internal static class Program { - private const string Url = "https://store.mall.autohome.com.cn/83106681.html"; - private const int ChromiumRevision = BrowserFetcher.DefaultRevision; + private const string Url = "https://github.com/orgs/dotnet/repositories"; private static async Task Main(string[] args) { - //Download chromium browser revision package - await new BrowserFetcher().DownloadAsync(ChromiumRevision); + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + // Download chromium browser revision package + await new BrowserFetcher().DownloadAsync(Chrome.DefaultBuildId); - //Test AngleSharp await TestAngleSharp(); - Console.ReadKey(); + if (ShouldWaitForExit()) + { + Console.WriteLine("Press any key to exit..."); + Console.ReadKey(intercept: true); + } + } + + private static bool ShouldWaitForExit() + { + return Environment.UserInteractive + && !Console.IsInputRedirected + && !Console.IsOutputRedirected; } private static async Task TestAngleSharp() { /* * Used AngleSharp loading of HTML document - * TODO: Used WithJavaScript function need install AngleSharp.Scripting.Javascript nuget package - * Note: that JavaScripts support is an experimental and does not support complex JavaScripts code. + * TODO: To enable JS evaluation, install the AngleSharp.Js nuget package + * (AngleSharp.Scripting.Javascript is deprecated and replaced by AngleSharp.Js) + * and call WithJs() on the configuration. + * Note: AngleSharp.Js (via Jint) now supports modern ECMAScript (ES2015–ES2025) and can run + * libraries such as jQuery / React 16 / Bootstrap 5 in its test suite. For many + * real-world pages you typically also need AngleSharp.Io, AngleSharp.Css, and WithEventLoop(). + * It is still not a full browser: DOM/browser API coverage and heavy SPA apps can fail, + * and Jint is an interpreter (slower than Chromium). Prefer PuppeteerSharp for complex sites. */ - //IConfiguration config = Configuration.Default.WithDefaultLoader(); + //IConfiguration config = Configuration.Default + // .WithDefaultLoader(new LoaderOptions { IsResourceLoadingEnabled = true }) + // //.WithCss() + // .WithJs() + // .WithEventLoop(); //IBrowsingContext context = BrowsingContext.New(config); //IDocument document = await context.OpenAsync(Url); - //Used PuppeteerSharp loading of HTML document - var htmlString = await TestPuppeteerSharp(); + // Used PuppeteerSharp loading of HTML document + string htmlString = await TestPuppeteerSharp(); /* * Parsing of HTML document string */ - var context = BrowsingContext.New(Configuration.Default); - var parser = context.GetService(); - var document = parser.ParseDocument(htmlString); + IBrowsingContext context = BrowsingContext.New(Configuration.Default); + IHtmlParser? parser = context.GetService(); + if (parser == null) + { + throw new InvalidOperationException($"Failed to get {nameof(IHtmlParser)} service."); + } - //Selector carbox element list - var carboxList = document.QuerySelectorAll("div.shop-content div.content div.list li.carbox"); + IHtmlDocument document = parser.ParseDocument(htmlString); - var carModelList = new List(); - foreach (var carbox in carboxList) + // Selector repository element list + // Note: GitHub UI markup changes over time; keep selectors under review when scraping breaks. + IHtmlCollection repoElementList = document + .QuerySelectorAll("ul[data-listview-component='items-list'] > li") + .OfType() + .ToCollection(); + + RepoModel[] repoModels = new RepoModel[repoElementList.Length]; + for (int i = 0; i < repoElementList.Length; i++) { - //Parsing and converting to the car model object. - var model = CreateModelWithAngleSharp(carbox); - carModelList.Add(model); - - //Printing to console windows - var jsonString = JsonConvert.SerializeObject(model); - Console.WriteLine(jsonString); - Console.WriteLine(); + // Parsing and converting to the repository model object. + repoModels[i] = CreateModelWithAngleSharp(repoElementList[i]); } - Console.WriteLine("Total count:" + carModelList.Count); + // Printing to console windows + JsonSerializerOptions options = new JsonSerializerOptions + { + TypeInfoResolver = CustomJsonSerializerContext.Default, + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + string jsonString = JsonSerializer.Serialize(repoModels, options); + Console.WriteLine(jsonString); + Console.WriteLine(); + Console.WriteLine("Total count: " + repoModels.Length); } private static async Task TestPuppeteerSharp() { - //Enabled headless option - var launchOptions = new LaunchOptions { Headless = true }; - //Starting headless browser - var browser = await Puppeteer.LaunchAsync(launchOptions); - - //Get all(default) pages - var pages = await browser.PagesAsync(); - //Get first page or new tab page - var firstPage = pages.Length > 0 ? pages[0]: await browser.NewPageAsync(); - //Request URL to get the page - await firstPage.GoToAsync(Url); - - //Get and return the HTML content of the page - var htmlString = await firstPage.GetContentAsync(); - - #region Dispose resources - //Close tab page - await firstPage.CloseAsync(); - - //Close headless browser, all pages will be closed here. - await browser.CloseAsync(); - #endregion - - return htmlString; + // Enabled headless option + LaunchOptions launchOptions = new LaunchOptions { Headless = true }; + // Starting headless browser + await using IBrowser browser = await Puppeteer.LaunchAsync(launchOptions).ConfigureAwait(false); + + // Get all(default) pages + IPage[] pages = await browser.PagesAsync().ConfigureAwait(false); + // Get first page or new tab page + IPage firstPage = pages.Length > 0 ? pages[0] : await browser.NewPageAsync().ConfigureAwait(false); + + try + { + // Request URL to get the page + await firstPage.GoToAsync(Url, WaitUntilNavigation.Networkidle2).ConfigureAwait(false); + // Get and return the HTML content of the page + return await firstPage.GetContentAsync().ConfigureAwait(false); + } + finally + { + if (!firstPage.IsClosed) + { + await firstPage.CloseAsync().ConfigureAwait(false); + } + } } - private static CarModel CreateModelWithAngleSharp(IParentNode node) + private static RepoModel CreateModelWithAngleSharp(IHtmlListItemElement repoItem) { - var model = new CarModel + return new RepoModel { - Title = node.QuerySelector("a div.carbox-title").TextContent, - ImageUrl = node.QuerySelector("a div.carbox-carimg img").GetAttribute("src"), - ProductUrl = node.QuerySelector("a").GetAttribute("href"), - Tip = node.QuerySelector("a div.carbox-tip").TextContent, - OrdersNumber = node.QuerySelector("a div.carbox-number span").TextContent + Url = NormalizeText(repoItem.QuerySelector("div[data-listview-item-title-container] > h4 > a")?.GetAttribute("href")), + Title = NormalizeText(repoItem.QuerySelector("div[data-listview-item-title-container] > h4 > a > span")?.TextContent), + Description = NormalizeText(repoItem.QuerySelector("div.repos-list-description > div")?.TextContent), + Visibility = NormalizeText(repoItem.QuerySelector("span[data-listview-item-visibility-label]")?.TextContent), + // CSS-module class prefixes are fragile; prefer data-* attributes when available. + Language = NormalizeText(repoItem.QuerySelector("span[class^='ReposListItem-module__PrimaryLanguageName']")?.TextContent), + License = NormalizeText(repoItem.QuerySelector("div[class^='ReposListItem-module__IconLabel']")?.TextContent) }; - - return model; } + + private static string? NormalizeText(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); } } diff --git a/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin64ReleaseToLocalFolder.pubxml b/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin64ReleaseToLocalFolder.pubxml index f113d4a..c2102a4 100644 --- a/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin64ReleaseToLocalFolder.pubxml +++ b/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin64ReleaseToLocalFolder.pubxml @@ -1,16 +1,17 @@  FileSystem Release Any CPU - netcoreapp3.0 - ../../build/win/netcoreapp3.0/x64 + net10.0 + ../../build/win/net10.0/x64 win-x64 true - <_IsPortable>false + true + true - \ No newline at end of file + diff --git a/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin86ReleaseToLocalFolder.pubxml b/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin86ReleaseToLocalFolder.pubxml index e6ff13a..b01b002 100644 --- a/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin86ReleaseToLocalFolder.pubxml +++ b/src/CrawlerSamples.ConsoleApp/Properties/PublishProfiles/PublishWin86ReleaseToLocalFolder.pubxml @@ -1,16 +1,17 @@  FileSystem Release Any CPU - netcoreapp3.0 - ../../build/win/netcoreapp3.0/x86 + net10.0 + ../../build/win/net10.0/x86 win-x86 true - <_IsPortable>false + true + true - \ No newline at end of file + diff --git a/src/CrawlerSamples.ConsoleApp/RepoModel.cs b/src/CrawlerSamples.ConsoleApp/RepoModel.cs new file mode 100644 index 0000000..bbe2fde --- /dev/null +++ b/src/CrawlerSamples.ConsoleApp/RepoModel.cs @@ -0,0 +1,26 @@ +using System.Text.Json.Serialization; + +namespace CrawlerSamples +{ + internal readonly record struct RepoModel + { + public string? Url { get; init; } + + public string? Visibility { get; init; } + + public string? Title { get; init; } + + public string? Description { get; init; } + + public string? Language { get; init; } + + public string? License { get; init; } + } + + [JsonSerializable(typeof(RepoModel))] + [JsonSerializable(typeof(RepoModel[]))] + internal partial class CustomJsonSerializerContext : JsonSerializerContext + { + + } +}