diff --git a/scripts/fetch_dependencies.ps1 b/scripts/fetch_dependencies.ps1 index dbcf507..bc41682 100644 --- a/scripts/fetch_dependencies.ps1 +++ b/scripts/fetch_dependencies.ps1 @@ -14,9 +14,16 @@ Set-Location $ExternalDir $OpenJTalkVersion = "1.11" $HTSEngineVersion = "1.10" -# Expected SHA256 checksums (update these when upgrading versions) -$HTSEngineSHA256 = "e2132be5860d8fb4a460be766454cfd7c3e21cf67b509c48e1804feab14968f7" -$OpenJTalkSHA256 = "20fdc6aeb6c757866034abc175820573db43e4284707c866fcd02c8ec18de71f" +# GitHub Release mirror for stable downloads (SourceForge hashes can change due to CDN re-compression) +$GitHubMirrorBase = "https://github.com/ayutaz/openjtalk-native/releases/download/deps-v1" + +# SHA256 checksums of .tar.gz files hosted on GitHub Release (stable) +$HTSEngineGzSHA256 = "e2132be5860d8fb4a460be766454cfd7c3e21cf67b509c48e1804feab14968f7" +$OpenJTalkGzSHA256 = "20fdc6aeb6c757866034abc175820573db43e4284707c866fcd02c8ec18de71f" + +# SHA256 checksums of inner tar content (stable across gzip re-compression) +$HTSEngineTarSHA256 = "d2924c144f3463b0caea21b6e03cd1573f945652e71f6f221642699e3acec1ff" +$OpenJTalkTarSHA256 = "1c80e8478e0fa8b8d19e404c897d943d1c496aa84862bf60f58c100c30dd4804" function Verify-Checksum { param( @@ -31,16 +38,62 @@ function Verify-Checksum { $actualHash = (Get-FileHash -Path $FilePath -Algorithm SHA256).Hash.ToLower() if ($actualHash -ne $ExpectedHash) { - Write-Host "ERROR: Checksum mismatch for $FilePath" + Write-Host "WARNING: Archive checksum mismatch for $FilePath (may be due to SourceForge re-compression)" Write-Host " Expected: $ExpectedHash" Write-Host " Actual: $actualHash" - Remove-Item $FilePath -Force return $false } Write-Host "Checksum verified for $FilePath" return $true } +function Verify-GzipFormat { + param([string]$FilePath) + + $bytes = [System.IO.File]::ReadAllBytes($FilePath) + if ($bytes.Length -lt 2 -or $bytes[0] -ne 0x1F -or $bytes[1] -ne 0x8B) { + Write-Host "ERROR: $FilePath is not a valid gzip file" + Remove-Item $FilePath -Force + return $false + } + return $true +} + +function Verify-TarChecksum { + param( + [string]$FilePath, + [string]$ExpectedHash + ) + + if ([string]::IsNullOrEmpty($ExpectedHash)) { + Write-Host "WARNING: No inner tar checksum provided, skipping for $FilePath" + return $true + } + + try { + $inStream = [System.IO.File]::OpenRead($FilePath) + $gzStream = New-Object System.IO.Compression.GZipStream($inStream, [System.IO.Compression.CompressionMode]::Decompress) + $sha256 = [System.Security.Cryptography.SHA256]::Create() + $hashBytes = $sha256.ComputeHash($gzStream) + $gzStream.Close() + $inStream.Close() + $actualHash = -join ($hashBytes | ForEach-Object { $_.ToString("x2") }) + + if ($actualHash -ne $ExpectedHash) { + Write-Host "ERROR: Inner tar checksum mismatch for $FilePath" + Write-Host " Expected: $ExpectedHash" + Write-Host " Actual: $actualHash" + Remove-Item $FilePath -Force + return $false + } + Write-Host "Inner tar checksum verified for $FilePath" + return $true + } catch { + Write-Host "WARNING: Failed to verify inner tar checksum: $_" + return $true + } +} + function Download-WithRetry { param( [string]$Url, @@ -66,6 +119,27 @@ function Download-WithRetry { return $false } +function Download-WithFallback { + param( + [string]$PrimaryUrl, + [string]$FallbackUrl, + [string]$Output + ) + + Write-Host "Trying primary mirror (GitHub)..." + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $PrimaryUrl -OutFile $Output -UseBasicParsing -TimeoutSec 120 + Write-Host "Downloaded from GitHub mirror." + return $true + } catch { + Write-Host "Primary mirror unavailable: $_" + } + + Write-Host "Trying SourceForge..." + return (Download-WithRetry -Url $FallbackUrl -Output $Output) +} + function Extract-TarGz { param([string]$Archive) @@ -88,20 +162,28 @@ Write-Host "=== Fetching OpenJTalk Dependencies ===" $HTSDir = "hts_engine_API-$HTSEngineVersion" if (-not (Test-Path $HTSDir)) { Write-Host "Downloading hts_engine_API $HTSEngineVersion..." - $result = Download-WithRetry ` - -Url "https://sourceforge.net/projects/hts-engine/files/hts_engine%20API/hts_engine_API-$HTSEngineVersion/hts_engine_API-$HTSEngineVersion.tar.gz/download" ` + $result = Download-WithFallback ` + -PrimaryUrl "$GitHubMirrorBase/hts_engine_API-$HTSEngineVersion.tar.gz" ` + -FallbackUrl "https://sourceforge.net/projects/hts-engine/files/hts_engine%20API/hts_engine_API-$HTSEngineVersion/hts_engine_API-$HTSEngineVersion.tar.gz/download" ` -Output "hts_engine_API.tar.gz" if (-not $result) { - Write-Error "Failed to download hts_engine_API after multiple attempts" + Write-Error "Failed to download hts_engine_API" exit 1 } - if (-not (Verify-Checksum "hts_engine_API.tar.gz" $HTSEngineSHA256)) { - Write-Error "Checksum verification failed for hts_engine_API" + if (-not (Verify-GzipFormat "hts_engine_API.tar.gz")) { + Write-Error "Downloaded hts_engine_API is not a valid gzip file" exit 1 } + if (-not (Verify-Checksum "hts_engine_API.tar.gz" $HTSEngineGzSHA256)) { + if (-not (Verify-TarChecksum "hts_engine_API.tar.gz" $HTSEngineTarSHA256)) { + Write-Error "Checksum verification failed for hts_engine_API" + exit 1 + } + } + Write-Host "Extracting hts_engine_API..." Extract-TarGz "hts_engine_API.tar.gz" Remove-Item "hts_engine_API.tar.gz" @@ -111,20 +193,28 @@ if (-not (Test-Path $HTSDir)) { $OJTDir = "open_jtalk-$OpenJTalkVersion" if (-not (Test-Path $OJTDir)) { Write-Host "Downloading OpenJTalk $OpenJTalkVersion..." - $result = Download-WithRetry ` - -Url "https://sourceforge.net/projects/open-jtalk/files/Open%20JTalk/open_jtalk-$OpenJTalkVersion/open_jtalk-$OpenJTalkVersion.tar.gz/download" ` + $result = Download-WithFallback ` + -PrimaryUrl "$GitHubMirrorBase/open_jtalk-$OpenJTalkVersion.tar.gz" ` + -FallbackUrl "https://sourceforge.net/projects/open-jtalk/files/Open%20JTalk/open_jtalk-$OpenJTalkVersion/open_jtalk-$OpenJTalkVersion.tar.gz/download" ` -Output "open_jtalk.tar.gz" if (-not $result) { - Write-Error "Failed to download OpenJTalk after multiple attempts" + Write-Error "Failed to download OpenJTalk" exit 1 } - if (-not (Verify-Checksum "open_jtalk.tar.gz" $OpenJTalkSHA256)) { - Write-Error "Checksum verification failed for OpenJTalk" + if (-not (Verify-GzipFormat "open_jtalk.tar.gz")) { + Write-Error "Downloaded OpenJTalk is not a valid gzip file" exit 1 } + if (-not (Verify-Checksum "open_jtalk.tar.gz" $OpenJTalkGzSHA256)) { + if (-not (Verify-TarChecksum "open_jtalk.tar.gz" $OpenJTalkTarSHA256)) { + Write-Error "Checksum verification failed for OpenJTalk" + exit 1 + } + } + Write-Host "Extracting OpenJTalk..." Extract-TarGz "open_jtalk.tar.gz" Remove-Item "open_jtalk.tar.gz" diff --git a/scripts/fetch_dependencies.sh b/scripts/fetch_dependencies.sh index bbe0a3c..6c69d66 100644 --- a/scripts/fetch_dependencies.sh +++ b/scripts/fetch_dependencies.sh @@ -32,6 +32,20 @@ download_with_retry() { return 1 } +download_with_fallback() { + local primary_url="$1" + local fallback_url="$2" + local output="$3" + + echo "Trying primary mirror (GitHub)..." + if curl -L --fail --connect-timeout 15 --max-time 120 -o "$output" "$primary_url" 2>/dev/null; then + echo "Downloaded from GitHub mirror." + return 0 + fi + echo "Primary mirror unavailable, trying SourceForge..." + download_with_retry "$fallback_url" "$output" +} + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT_DIR="$(dirname "$SCRIPT_DIR")" EXTERNAL_DIR="$ROOT_DIR/external" @@ -44,9 +58,16 @@ cd "$EXTERNAL_DIR" OPENJTALK_VERSION="1.11" HTS_ENGINE_VERSION="1.10" -# Expected SHA256 checksums (update these when upgrading versions) -HTS_ENGINE_SHA256="e2132be5860d8fb4a460be766454cfd7c3e21cf67b509c48e1804feab14968f7" -OPENJTALK_SHA256="20fdc6aeb6c757866034abc175820573db43e4284707c866fcd02c8ec18de71f" +# GitHub Release mirror for stable downloads (SourceForge hashes can change due to CDN re-compression) +GITHUB_MIRROR_BASE="https://github.com/ayutaz/openjtalk-native/releases/download/deps-v1" + +# SHA256 checksums of .tar.gz files hosted on GitHub Release (stable) +HTS_ENGINE_GZ_SHA256="e2132be5860d8fb4a460be766454cfd7c3e21cf67b509c48e1804feab14968f7" +OPENJTALK_GZ_SHA256="20fdc6aeb6c757866034abc175820573db43e4284707c866fcd02c8ec18de71f" + +# SHA256 checksums of inner tar content (stable across gzip re-compression) +HTS_ENGINE_TAR_SHA256="d2924c144f3463b0caea21b6e03cd1573f945652e71f6f221642699e3acec1ff" +OPENJTALK_TAR_SHA256="1c80e8478e0fa8b8d19e404c897d943d1c496aa84862bf60f58c100c30dd4804" verify_checksum() { local file="$1" @@ -65,27 +86,70 @@ verify_checksum() { return 0 fi if [ "$actual" != "$expected" ]; then - echo "ERROR: Checksum mismatch for $file" + echo "WARNING: Archive checksum mismatch for $file (may be due to SourceForge re-compression)" echo " Expected: $expected" echo " Actual: $actual" - rm -f "$file" return 1 fi echo "Checksum verified for $file" return 0 } +verify_gzip_format() { + local file="$1" + local magic + magic=$(od -A n -N 2 -t x1 "$file" 2>/dev/null | tr -d ' \n') + if [ "$magic" != "1f8b" ]; then + echo "ERROR: $file is not a valid gzip file (magic: $magic)" + rm -f "$file" + return 1 + fi + return 0 +} + +verify_tar_checksum() { + local file="$1" + local expected="$2" + if [ -z "$expected" ]; then + echo "WARNING: No inner tar checksum provided, skipping for $file" + return 0 + fi + local actual + if command -v sha256sum >/dev/null 2>&1; then + actual=$(gzip -dc "$file" | sha256sum | awk '{print $1}') + elif command -v shasum >/dev/null 2>&1; then + actual=$(gzip -dc "$file" | shasum -a 256 | awk '{print $1}') + else + echo "WARNING: No SHA256 tool found, skipping inner tar verification" + return 0 + fi + if [ "$actual" != "$expected" ]; then + echo "ERROR: Inner tar checksum mismatch for $file" + echo " Expected: $expected" + echo " Actual: $actual" + rm -f "$file" + return 1 + fi + echo "Inner tar checksum verified for $file" + return 0 +} + echo "=== Fetching OpenJTalk Dependencies ===" # Download hts_engine_API if [ ! -d "hts_engine_API-${HTS_ENGINE_VERSION}" ]; then echo "Downloading hts_engine_API ${HTS_ENGINE_VERSION}..." - download_with_retry \ + download_with_fallback \ + "${GITHUB_MIRROR_BASE}/hts_engine_API-${HTS_ENGINE_VERSION}.tar.gz" \ "https://sourceforge.net/projects/hts-engine/files/hts_engine%20API/hts_engine_API-${HTS_ENGINE_VERSION}/hts_engine_API-${HTS_ENGINE_VERSION}.tar.gz/download" \ "hts_engine_API.tar.gz" || \ - error_exit "Failed to download hts_engine_API after multiple attempts" + error_exit "Failed to download hts_engine_API" - verify_checksum "hts_engine_API.tar.gz" "$HTS_ENGINE_SHA256" || \ + verify_gzip_format "hts_engine_API.tar.gz" || \ + error_exit "Downloaded hts_engine_API is not a valid gzip file" + + verify_checksum "hts_engine_API.tar.gz" "$HTS_ENGINE_GZ_SHA256" || \ + verify_tar_checksum "hts_engine_API.tar.gz" "$HTS_ENGINE_TAR_SHA256" || \ error_exit "Checksum verification failed for hts_engine_API" echo "Extracting hts_engine_API..." @@ -96,12 +160,17 @@ fi # Download OpenJTalk if [ ! -d "open_jtalk-${OPENJTALK_VERSION}" ]; then echo "Downloading OpenJTalk ${OPENJTALK_VERSION}..." - download_with_retry \ + download_with_fallback \ + "${GITHUB_MIRROR_BASE}/open_jtalk-${OPENJTALK_VERSION}.tar.gz" \ "https://sourceforge.net/projects/open-jtalk/files/Open%20JTalk/open_jtalk-${OPENJTALK_VERSION}/open_jtalk-${OPENJTALK_VERSION}.tar.gz/download" \ "open_jtalk.tar.gz" || \ - error_exit "Failed to download OpenJTalk after multiple attempts" + error_exit "Failed to download OpenJTalk" + + verify_gzip_format "open_jtalk.tar.gz" || \ + error_exit "Downloaded OpenJTalk is not a valid gzip file" - verify_checksum "open_jtalk.tar.gz" "$OPENJTALK_SHA256" || \ + verify_checksum "open_jtalk.tar.gz" "$OPENJTALK_GZ_SHA256" || \ + verify_tar_checksum "open_jtalk.tar.gz" "$OPENJTALK_TAR_SHA256" || \ error_exit "Checksum verification failed for OpenJTalk" echo "Extracting OpenJTalk..."