diff --git a/.gitignore b/.gitignore index c93362875..60e0dc641 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ tools/authenticator/.proxies.txt.swp /target/ /bin/ .claude -.gocache \ No newline at end of file +.gocache +.atomcode +CLAUDE.md \ No newline at end of file diff --git a/conversion/response/chatgpt/convert.go b/conversion/response/chatgpt/convert.go index dc57c41c4..aec15ecbe 100644 --- a/conversion/response/chatgpt/convert.go +++ b/conversion/response/chatgpt/convert.go @@ -9,12 +9,22 @@ import ( ) func ConvertToString(chatgpt_response *chatgpt_types.ChatGPTResponse, previous_text *typings.StringStruct, role bool, model string) string { - translated_response := official_types.NewChatCompletionChunk(strings.Replace(chatgpt_response.Message.Content.Parts[0].(string), previous_text.Text, "", 1), model) + currentText := firstTextPart(chatgpt_response.Message.Content.Parts) + deltaText := strings.Replace(currentText, previous_text.Text, "", 1) + translated_response := official_types.NewChatCompletionChunk(deltaText, model) if role { translated_response.Choices[0].Delta.Role = chatgpt_response.Message.Author.Role - } else if translated_response.Choices[0].Delta.Content == "" || (strings.HasPrefix(chatgpt_response.Message.Metadata.ModelSlug, "gpt-4") && translated_response.Choices[0].Delta.Content == "【") { + } else if translated_response.Choices[0].Delta.Content == "" || translated_response.Choices[0].Delta.Content == "【" { return translated_response.Choices[0].Delta.Content } - previous_text.Text = chatgpt_response.Message.Content.Parts[0].(string) + previous_text.Text = currentText return "data: " + translated_response.String() + "\n\n" } + +func firstTextPart(parts []interface{}) string { + if len(parts) == 0 { + return "" + } + text, _ := parts[0].(string) + return text +} diff --git a/conversion/response/chatgpt/convert_test.go b/conversion/response/chatgpt/convert_test.go new file mode 100644 index 000000000..62a3dceaa --- /dev/null +++ b/conversion/response/chatgpt/convert_test.go @@ -0,0 +1,52 @@ +package chatgpt + +import ( + "encoding/json" + "testing" + + "aurora/typings" + chatgpt_types "aurora/typings/chatgpt" +) + +func TestConvertToStringUsesRequestModel(t *testing.T) { + previous := &typings.StringStruct{} + response := chatgpt_types.ChatGPTResponse{ + Message: chatgpt_types.Message{ + Author: chatgpt_types.Author{Role: "assistant"}, + Content: chatgpt_types.Content{ + Parts: []interface{}{"hello"}, + }, + }, + } + + data := ConvertToString(&response, previous, true, "gpt-5-5-pro") + + var event struct { + Model string `json:"model"` + } + if err := json.Unmarshal([]byte(data[len("data: "):len(data)-2]), &event); err != nil { + t.Fatalf("invalid SSE JSON: %v", err) + } + if event.Model != "gpt-5-5-pro" { + t.Fatalf("model = %q, want request model", event.Model) + } +} + +func TestConvertToStringWaitsSourceMarkerForAnyModel(t *testing.T) { + previous := &typings.StringStruct{Text: "answer"} + response := chatgpt_types.ChatGPTResponse{ + Message: chatgpt_types.Message{ + Author: chatgpt_types.Author{Role: "assistant"}, + Content: chatgpt_types.Content{ + Parts: []interface{}{"answer【"}, + }, + Metadata: chatgpt_types.Metadata{ModelSlug: "gpt-5-5-pro"}, + }, + } + + data := ConvertToString(&response, previous, false, "gpt-5-5-pro") + + if data != "【" { + t.Fatalf("data = %q, want source marker only", data) + } +} diff --git a/go.mod b/go.mod index e72fadd24..65f28c447 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/joho/godotenv v1.5.1 github.com/pkoukk/tiktoken-go v0.1.7 - github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c golang.org/x/crypto v0.52.0 ) diff --git a/go.sum b/go.sum index c2bca233c..3c40eba53 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c h1:nj17XsSTwprsZUDXLldOUZmqz7VlHsLCeXXFOE6Q+Mk= -github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c/go.mod h1:7aCyoW5MHDUsoooMVLqKe0F7W9HMPUvDG3bXqw++8XA= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/initialize/handlers.go b/initialize/handlers.go index 55b65d39f..3e1da5eae 100644 --- a/initialize/handlers.go +++ b/initialize/handlers.go @@ -951,29 +951,29 @@ func (h *Handler) tts(c *gin.Context) { } client := bogdanfinn.NewStdClient() - turnStile, status, err := h.initTurnStileWithRetry(&client, &secret, proxyUrl) - if err != nil { - c.JSON(status, gin.H{ - "message": err.Error(), - "type": "InitTurnStile_request_error", - "param": err, - "code": status, - }) - return - } // Convert the chat request to a ChatGPT request translated_request := chatgptrequestconverter.ConvertTTSAPIRequest(original_request.Input) + clientState := chatgpt.NewChatClientState() + clientState.ConversationID = translated_request.ConversationID + clientState.ParentMessageID = translated_request.ParentMessageID - response, err := chatgpt.POSTconversation(client, translated_request, secret, turnStile, proxyUrl) + response, wsConn, status, err := h.postConversationGptClientOrder(&client, &secret, translated_request, proxyUrl, false, clientState) if err != nil { - c.JSON(500, gin.H{ - "error": "error sending request", - }) + c.JSON(status, gin.H{"error": gin.H{ + "message": err.Error(), + "type": "request_conversion_error", + "param": "model", + "code": "request_conversion_error", + }}) return } defer response.Body.Close() if chatgpt.Handle_request_error(c, response) { + if wsConn != nil { + wsConn.Close() + wsConn = nil + } return } msgId, convId := chatgpt.HandlerTTS(response, original_request.Input) diff --git a/internal/chatgpt/artifact_delivery.go b/internal/chatgpt/artifact_delivery.go index 0c8e15c8a..cf73e344c 100644 --- a/internal/chatgpt/artifact_delivery.go +++ b/internal/chatgpt/artifact_delivery.go @@ -238,7 +238,7 @@ func DownloadURLBytes(client httpclient.AuroraHttpClient, url string, secret *to if secret != nil && secret.PUID != "" { header.Set("Cookie", "_puid="+secret.PUID+";") } - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) if accept == "" { accept = "*/*" } diff --git a/internal/chatgpt/client_state.go b/internal/chatgpt/client_state.go index 354dc1763..42d449fcc 100644 --- a/internal/chatgpt/client_state.go +++ b/internal/chatgpt/client_state.go @@ -4,9 +4,10 @@ import ( "math" "time" - chatgpt_types "aurora/typings/chatgpt" - + browser "github.com/EDDYCJY/fake-useragent" "github.com/google/uuid" + + chatgpt_types "aurora/typings/chatgpt" ) type ChatClientState struct { @@ -16,6 +17,7 @@ type ChatClientState struct { ConversationID string ParentMessageID string TurnCount int + UserAgent string } func NewChatClientState() *ChatClientState { @@ -24,6 +26,7 @@ func NewChatClientState() *ChatClientState { SessionID: uuid.NewString(), StartTime: time.Now(), ParentMessageID: "client-created-root", + UserAgent: browser.Random(), } } diff --git a/internal/chatgpt/files.go b/internal/chatgpt/files.go index 109747ca0..387a8d8dd 100644 --- a/internal/chatgpt/files.go +++ b/internal/chatgpt/files.go @@ -152,7 +152,7 @@ func putUpload(client httpclient.AuroraHttpClient, uploadURL, contentType string header.Set("x-ms-version", "2020-04-08") header.Set("Origin", "https://chatgpt.com") header.Set("Referer", "https://chatgpt.com/") - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "application/json, text/plain, */*") header.Set("Accept-Language", "en-US,en;q=0.8") response, err := client.Request(http.MethodPut, uploadURL, header, nil, bytes.NewReader(data)) diff --git a/internal/chatgpt/request.go b/internal/chatgpt/request.go index 4938fb5b3..21acf85bb 100644 --- a/internal/chatgpt/request.go +++ b/internal/chatgpt/request.go @@ -54,7 +54,6 @@ func init() { var ( API_REVERSE_PROXY = os.Getenv("API_REVERSE_PROXY") FILES_REVERSE_PROXY = os.Getenv("FILES_REVERSE_PROXY") - userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 Edg/147.0.0.0" oaiDeviceID = uuid.NewString() oaiSessionID = uuid.NewString() oaiStartTime = time.Now() @@ -134,11 +133,15 @@ func GetInitConfig() []interface{} { timeNum := float64(time.Since(oaiStartTime).Milliseconds()) loc := time.FixedZone("Eastern Standard Time", -5*60*60) parseTime := time.Now().In(loc).Format("Mon Jan 02 2006 15:04:05") + " GMT-0500 (Eastern Standard Time)" - return []interface{}{cachedHardware, parseTime, int64(4294705152), 0, userAgent, script, cachedDpl, "zh-CN", "zh-CN", 0, "webkitGetUserMedia−function webkitGetUserMedia() { [native code] }", "location", "ontransitionend", timeNum, uuid.NewString()} + return []interface{}{cachedHardware, parseTime, int64(4294705152), 0, defaultUserAgent(), script, cachedDpl, "zh-CN", "zh-CN", 0, "webkitGetUserMedia−function webkitGetUserMedia() { [native code] }", "location", "ontransitionend", timeNum, uuid.NewString()} } -func CalcProofToken(require *ChatRequire) string { - return prooftoken.CalcProofToken(require.Proof.Seed, require.Proof.Difficulty, userAgent) +func CalcProofToken(require *ChatRequire, state *ChatClientState) string { + ua := defaultUserAgent() + if state != nil && state.UserAgent != "" { + ua = state.UserAgent + } + return prooftoken.SolveProofToken(require.Proof.Seed, require.Proof.Difficulty, ua) } type ChatRequire struct { @@ -175,7 +178,11 @@ func InitSentinelWithState(client httpclient.AuroraHttpClient, secret *tokens.Se if proxy != "" { client.SetProxy(proxy) } - requirementsToken := prooftoken.LegacyRequirementsToken(userAgent) + ua := defaultUserAgent() + if state != nil && state.UserAgent != "" { + ua = state.UserAgent + } + requirementsToken := prooftoken.NewPOWConfig(ua).RequirementsToken() prepare, status, err := POSTSentinelPrepareWithState(client, secret, requirementsToken, state) if err != nil { if secret.IsFree && status == http.StatusUnauthorized && retry < 2 { @@ -202,7 +209,7 @@ func InitSentinelWithState(client httpclient.AuroraHttpClient, secret *tokens.Se var proofToken string if prepare.Proof.Required { - proofToken = CalcProofToken(prepare) + proofToken = CalcProofToken(prepare, state) if proofToken == "" { return nil, http.StatusForbidden, errors.New("calculation proof token failure. Please retry the operation") } @@ -337,7 +344,7 @@ func POSTTurnStile(client httpclient.AuroraHttpClient, secret *tokens.Secret, pr client.SetProxy(proxy) } if cachedRequireProof == "" { - cachedRequireProof = prooftoken.LegacyRequirementsToken(userAgent) + cachedRequireProof = prooftoken.NewPOWConfig(defaultUserAgent()).RequirementsToken() } var apiUrl string if secret.IsFree { @@ -583,7 +590,11 @@ func DialChatWebsocketWithStateAndProxy(client httpclient.AuroraHttpClient, secr dialer.Proxy = proxyFunc } header := http.Header{} - header.Set("User-Agent", userAgent) + ua := defaultUserAgent() + if state != nil && state.UserAgent != "" { + ua = state.UserAgent + } + header.Set("User-Agent", ua) header.Set("Origin", "https://chatgpt.com") conn, _, err := dialer.Dial(wsURL, header) if err != nil { @@ -973,7 +984,7 @@ func GETengines(client httpclient.AuroraHttpClient, secret *tokens.Secret, proxy reqUrl := BaseURL + "/models" header := make(httpclient.AuroraHeaders) header.Set("Content-Type", "application/json") - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "*/*") header.Set("oai-language", "en-US") header.Set("origin", "https://chatgpt.com") @@ -1548,7 +1559,7 @@ func GetImageSource(client httpclient.AuroraHttpClient, wg *sync.WaitGroup, url if secret != nil && secret.PUID != "" { header.Set("Cookie", "_puid="+secret.PUID+";") } - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "*/*") if secret != nil && secret.Token != "" { header.Set("Authorization", "Bearer "+secret.Token) @@ -1572,7 +1583,7 @@ func GetImageDownloadURL(client httpclient.AuroraHttpClient, url string, secret if secret != nil && secret.PUID != "" { header.Set("Cookie", "_puid="+secret.PUID+";") } - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "*/*") if secret != nil && secret.Token != "" { header.Set("Authorization", "Bearer "+secret.Token) @@ -1604,7 +1615,7 @@ func DownloadImageBytes(client httpclient.AuroraHttpClient, url string, secret * if secret != nil && secret.PUID != "" { header.Set("Cookie", "_puid="+secret.PUID+";") } - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8") if secret != nil && secret.Token != "" { header.Set("Authorization", "Bearer "+secret.Token) @@ -2070,6 +2081,9 @@ func HandlerDetailedWithWebsocket(c *gin.Context, response *http.Response, clien } func HandlerDetailedWithOptions(c *gin.Context, response *http.Response, client httpclient.AuroraHttpClient, secret *tokens.Secret, uuid string, translated_request chatgpt_types.ChatGPTRequest, stream bool, model string, options HandlerDetailedOptions) HandlerResult { + if model == "" { + model = translated_request.Model + } wsConn := options.Websocket if options.ClientState != nil { options.ClientState.ApplyToRequest(&translated_request) @@ -2546,7 +2560,7 @@ func GETTokenForSessionToken(client httpclient.AuroraHttpClient, session_token s header := make(httpclient.AuroraHeaders) header.Set("authority", "chat.openai.com") header.Set("accept-language", "zh-CN,zh;q=0.9") - header.Set("User-Agent", userAgent) + header.Set("User-Agent", defaultUserAgent()) header.Set("Accept", "*/*") header.Set("oai-language", "en-US") header.Set("origin", "https://chatgpt.com") @@ -2600,7 +2614,11 @@ func createBaseHeaderForState(state *ChatClientState) httpclient.AuroraHeaders { header.Set("sec-fetch-dest", "empty") header.Set("sec-fetch-mode", "cors") header.Set("sec-fetch-site", "same-origin") - header.Set("user-agent", userAgent) + ua := defaultUserAgent() + if state != nil && state.UserAgent != "" { + ua = state.UserAgent + } + header.Set("user-agent", ua) deviceID := oaiDeviceID sessionID := oaiSessionID if state != nil { @@ -2618,45 +2636,62 @@ func createBaseHeaderForState(state *ChatClientState) httpclient.AuroraHeaders { return header } +func defaultUserAgent() string { + return util.RandomUserAgent() +} + func HandlerTTS(response *http.Response, input string) (string, string) { reader := bufio.NewReader(response.Body) - var original_response chatgpt_types.ChatGPTResponse var convId string var fallbackMsgID string + var patchState conversationPatchState for { line, err := reader.ReadString('\n') if err != nil { - if err == io.EOF { + if err == io.EOF && line == "" { break } - return "", "" - } - if len(line) < 6 { - continue + if err != io.EOF { + return "", "" + } } - line = line[6:] - if !strings.HasPrefix(line, "[DONE]") { - original_response.Message.ID = "" - err = json.Unmarshal([]byte(line), &original_response) - if err != nil { + for _, payload := range sseDataPayloads(line) { + if strings.HasPrefix(payload, "[DONE]") { + break + } + streamEvent, ok := parseConversationEvent(payload, &patchState, "auto") + if !ok { + var raw map[string]interface{} + if json.Unmarshal([]byte(payload), &raw) == nil { + if cid := firstConversationID(raw); cid != "" && convId == "" { + convId = cid + } + if msgID := lastAssistantMessageID(raw); msgID != "" && fallbackMsgID == "" { + fallbackMsgID = msgID + } + } continue } - if original_response.Error != nil { + if streamEvent.response.Error != nil { return "", "" } - if original_response.Message.ID == "" { - continue + originalResponse := streamEvent.response + if streamEvent.conversationID != "" && convId == "" { + convId = streamEvent.conversationID } - if original_response.ConversationID != convId { + if originalResponse.ConversationID != convId { if convId == "" { - convId = original_response.ConversationID + convId = originalResponse.ConversationID } else { continue } } - if original_response.Message.Author.Role != "assistant" { + if originalResponse.Message.ID == "" { + continue + } + if originalResponse.Message.Author.Role != "assistant" { continue } @@ -2664,21 +2699,24 @@ func HandlerTTS(response *http.Response, input string) (string, string) { // requested TTS input. Prefer an exact match, then fall back to the first // assistant message in the same conversation so synthesize still works. if fallbackMsgID == "" { - fallbackMsgID = original_response.Message.ID + fallbackMsgID = originalResponse.Message.ID } - if len(original_response.Message.Content.Parts) == 0 { + if len(originalResponse.Message.Content.Parts) == 0 { continue } - for _, rawPart := range original_response.Message.Content.Parts { + for _, rawPart := range originalResponse.Message.Content.Parts { part, ok := rawPart.(string) if !ok { continue } if part == input || strings.Contains(part, input) || strings.Contains(input, part) { - return original_response.Message.ID, convId + return originalResponse.Message.ID, convId } } } + if err == io.EOF { + break + } } if fallbackMsgID != "" && convId != "" { return fallbackMsgID, convId diff --git a/internal/chatgpt/request_test.go b/internal/chatgpt/request_test.go index 546817eab..d4c0a588c 100644 --- a/internal/chatgpt/request_test.go +++ b/internal/chatgpt/request_test.go @@ -96,8 +96,9 @@ func TestHandlerStreamsOpenAIChunksAndSentinel(t *testing.T) { response := &http.Response{Body: io.NopCloser(strings.NewReader(body))} writer := httptest.NewRecorder() c, _ := gin.CreateTestContext(writer) + request := chatGPTRequestForTest() - full, continueInfo := Handler(c, response, nil, nil, "request-id", chatGPTRequestForTest(), true, "gpt-4o") + full, continueInfo := Handler(c, response, nil, nil, "request-id", request, true, request.Model) if continueInfo != nil { t.Fatalf("continueInfo = %#v, want nil", continueInfo) @@ -277,8 +278,9 @@ func TestHandlerDetailedCollectsOpenAIChunkMetadataWithoutStreaming(t *testing.T response := &http.Response{Body: io.NopCloser(strings.NewReader(body))} writer := httptest.NewRecorder() c, _ := gin.CreateTestContext(writer) + request := chatGPTRequestForTest() - result := HandlerDetailed(c, response, nil, nil, "request-id", chatGPTRequestForTest(), false, "gpt-4o") + result := HandlerDetailed(c, response, nil, nil, "request-id", request, false, request.Model) if result.Text != "你好" { t.Fatalf("text = %q, want %q", result.Text, "你好") @@ -587,6 +589,39 @@ func TestHandlerSeparatesAnalysisAndFinalChannels(t *testing.T) { } } +func TestHandlerTTSParsesPatchStream(t *testing.T) { + body := strings.Join([]string{ + `data: {"p":"/conversation_id","o":"replace","v":"conv-tts"}`, + `data: {"p":"/message/id","o":"replace","v":"msg-tts"}`, + `data: {"p":"/message/author/role","o":"replace","v":"assistant"}`, + `data: {"p":"/message/content/parts/0","o":"append","v":"hello tts"}`, + `data: [DONE]`, + ``, + }, "\n") + response := &http.Response{Body: io.NopCloser(strings.NewReader(body))} + + msgID, convID := HandlerTTS(response, "hello tts") + + if msgID != "msg-tts" || convID != "conv-tts" { + t.Fatalf("msgID=%q convID=%q, want msg-tts conv-tts", msgID, convID) + } +} + +func TestHandlerTTSFallsBackToAssistantMessageID(t *testing.T) { + body := strings.Join([]string{ + `data: {"conversation_id":"conv-tts","message":{"id":"msg-tts","author":{"role":"assistant"},"content":{"content_type":"text","parts":["different text"]},"metadata":{"message_type":"next"},"recipient":"all"}}`, + `data: [DONE]`, + ``, + }, "\n") + response := &http.Response{Body: io.NopCloser(strings.NewReader(body))} + + msgID, convID := HandlerTTS(response, "requested text") + + if msgID != "msg-tts" || convID != "conv-tts" { + t.Fatalf("msgID=%q convID=%q, want fallback assistant message", msgID, convID) + } +} + func chatGPTRequestForTest() chatgpt.ChatGPTRequest { return chatgpt.NewChatGPTRequest() } diff --git a/internal/prooftoken/prooftoken.go b/internal/prooftoken/prooftoken.go index 15eacd13a..b9075b3e8 100644 --- a/internal/prooftoken/prooftoken.go +++ b/internal/prooftoken/prooftoken.go @@ -7,252 +7,237 @@ import ( "encoding/json" "fmt" "math/rand" - "regexp" "strconv" "strings" + "sync/atomic" "time" - "github.com/google/uuid" "golang.org/x/crypto/sha3" ) const ( - defaultPowScript = "https://chatgpt.com/backend-api/sentinel/sdk.js" - maxAttempts = 500000 + powPrefixRequirements = "gAAAAAC" + powPrefixProof = "gAAAAAB" + + requirementsDifficulty = "0fffff" + + maxRequirementsIter = 500_000 + maxProofIter = 100_000 + + powFallback = "gAAAAABwQ8Lk5FbGpA2NcR9dShT6gYjU7VxZ4D" ) var ( - cores = []int{8, 16, 24, 32} - screenValues = []int{3000, 4000, 5000} - documentKeys = []string{ - "_reactListeningo743lnnpvdg", - "location", - } - navigatorKeys = []string{ - "registerProtocolHandler−function registerProtocolHandler() { [native code] }", - "storage−[object StorageManager]", - "locks−[object LockManager]", - "appCodeName−Mozilla", - "permissions−[object Permissions]", - "share−function share() { [native code] }", - "webdriver−false", - "managed−[object NavigatorManagedData]", - "canShare−function canShare() { [native code] }", - "vendor−Google Inc.", - "mediaDevices−[object MediaDevices]", - "vibrate−function vibrate() { [native code] }", - "storageBuckets−[object StorageBucketManager]", - "mediaCapabilities−[object MediaCapabilities]", - "cookieEnabled−true", - "virtualKeyboard−[object VirtualKeyboard]", - "product−Gecko", - "presentation−[object Presentation]", - "onLine−true", - "mimeTypes−[object MimeTypeArray]", - "credentials−[object CredentialsContainer]", - "serviceWorker−[object ServiceWorkerContainer]", - "keyboard−[object Keyboard]", - "gpu−[object GPU]", - "doNotTrack", - "serial−[object Serial]", - "pdfViewerEnabled−true", - "language−zh-CN", - "geolocation−[object Geolocation]", - "userAgentData−[object NavigatorUAData]", - "getUserMedia−function getUserMedia() { [native code] }", - "sendBeacon−function sendBeacon() { [native code] }", - "hardwareConcurrency−32", - "windowControlsOverlay−[object WindowControlsOverlay]", + powCores = []int{16, 24, 32} + powScreens = []int{3000, 4000, 6000} + + powNavKeys = []string{ + "webdriver-false", "vendor-Google Inc.", "cookieEnabled-true", + "pdfViewerEnabled-true", "hardwareConcurrency-32", + "language-zh-CN", "mimeTypes-[object MimeTypeArray]", + "userAgentData-[object NavigatorUAData]", } - windowKeys = []string{ - "0", - "window", - "self", - "document", - "name", - "location", - "customElements", - "history", - "navigation", - "innerWidth", - "innerHeight", - "scrollX", - "scrollY", - "visualViewport", - "screenX", - "screenY", - "outerWidth", - "outerHeight", - "devicePixelRatio", - "screen", - "chrome", - "navigator", - "onresize", - "performance", - "crypto", - "indexedDB", - "sessionStorage", - "localStorage", - "scheduler", - "alert", - "atob", - "btoa", - "fetch", - "matchMedia", - "postMessage", - "queueMicrotask", - "requestAnimationFrame", - "setInterval", - "setTimeout", - "caches", - "__NEXT_DATA__", - "__BUILD_MANIFEST", - "__NEXT_PRELOADREADY", + powWinKeys = []string{ + "innerWidth", "innerHeight", "devicePixelRatio", "screen", + "chrome", "location", "history", "navigator", } - scriptSrcRE = regexp.MustCompile(`]*\bsrc=["']([^"']+)["']`) - dataBuildRE = regexp.MustCompile(`(?:c/[^/]*/_|]*data-build=["']([^"']*)["'])`) -) -type ProofWork struct { - Difficulty string `json:"difficulty,omitempty"` - Required bool `json:"required"` - Seed string `json:"seed,omitempty"` - Ospt string `json:"-"` -} + powReactListeners = []string{"_reactListeningcfilawjnerp", "_reactListening9ne2dfo1i47"} + powProofEvents = []string{"alert", "ontransitionend", "onprogress"} -type Resources struct { - ScriptSources []string - DataBuild string + // perfCounter 模拟浏览器 performance.counter() 的单调递增(亚秒级)。 + perfCounter uint64 +) + +// POWConfig 是 18 元素的客户端指纹数组(requirements_token 用)。 +type POWConfig struct { + userAgent string + arr [18]interface{} } -func ParseResources(html string) Resources { - resources := Resources{} - for _, match := range scriptSrcRE.FindAllStringSubmatch(html, -1) { - resources.ScriptSources = append(resources.ScriptSources, match[1]) - if resources.DataBuild == "" { - if build := regexp.MustCompile(`c/[^/]*/_`).FindString(match[1]); build != "" { - resources.DataBuild = build - } - } - } - if len(resources.ScriptSources) == 0 { - resources.ScriptSources = []string{defaultPowScript} - } - if resources.DataBuild == "" { - for _, match := range dataBuildRE.FindAllStringSubmatch(html, -1) { - if len(match) > 1 && match[1] != "" { - resources.DataBuild = match[1] - break - } - if match[0] != "" && strings.HasPrefix(match[0], "c/") { - resources.DataBuild = match[0] - break - } - } +// NewPOWConfig 构造一个随机化的客户端指纹,用于 requirements + proof 两种场景。 +func NewPOWConfig(userAgent string) *POWConfig { + if userAgent == "" { + userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 Edg/147.0.0.0" } - return resources + //nolint:gosec // 非加密用途 + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + now := time.Now().UTC() + timeStr := now.Format("Mon Jan 02 2006 15:04:05") + " GMT+0000 (UTC)" + perf := float64(atomic.AddUint64(&perfCounter, 1)) + rng.Float64() + + c := &POWConfig{userAgent: userAgent} + c.arr = [18]interface{}{ + powCores[rng.Intn(len(powCores))] + powScreens[rng.Intn(len(powScreens))], // 0 + timeStr, // 1 + nil, // 2 + rng.Float64(), // 3 - 迭代会覆盖 + userAgent, // 4 + nil, // 5 + "dpl=1440a687921de39ff5ee56b92807faaadce73f13", // 6 + "en-US", // 7 + "en-US,zh-CN", // 8 + 0, // 9 - 迭代会覆盖 + powNavKeys[rng.Intn(len(powNavKeys))], // 10 + "location", // 11 + powWinKeys[rng.Intn(len(powWinKeys))], // 12 + perf, // 13 + randomUUID(rng), // 14 + "", // 15 + 8, // 16 + now.Unix(), // 17 + } + return c } -func CalcProofToken(seed string, difficulty string, userAgent string, resources ...Resources) string { - answer, ok := generate(seed, difficulty, buildConfig(userAgent, firstResource(resources))) +// RequirementsToken 生成 /sentinel/chat-requirements 的 "p" 字段值。 +// 对齐 gen_image.py.get_requirements_token:固定难度 0fffff,前缀 gAAAAAC。 +func (c *POWConfig) RequirementsToken() string { + //nolint:gosec + seed := strconv.FormatFloat(rand.Float64(), 'f', -1, 64) + b64, ok := c.solveRequirements(seed, requirementsDifficulty) if !ok { - return "" + return powPrefixRequirements + powFallback + + base64.StdEncoding.EncodeToString([]byte(`"`+seed+`"`)) } - return "gAAAAAB" + answer + return powPrefixRequirements + b64 } -func LegacyRequirementsToken(userAgent string, resources ...Resources) string { - seed := fmt.Sprintf("%v", rand.New(rand.NewSource(time.Now().UnixNano())).Float64()) - answer, _ := generate(seed, "0fffff", buildConfig(userAgent, firstResource(resources))) - return "gAAAAAC" + answer -} +// solveRequirements 高性能迭代:预拼 JSON 的三段字节前缀,只在内循环拼 d1/d2。 +// 严格对齐 gen_image.py._generate_answer。 +func (c *POWConfig) solveRequirements(seed, difficulty string) (string, bool) { + target, err := hex.DecodeString(difficulty) + if err != nil { + return "", false + } + diffLen := len(difficulty) // 字符数(与 Python 对齐) + + // 预拼 p1/p2/p3。config[3] 和 config[9] 位置留给迭代器。 + arr := c.arr + // p1 = json(arr[:3])[:-1] + "," + head, _ := json.Marshal([]interface{}{arr[0], arr[1], arr[2]}) + p1 := append(head[:len(head)-1:len(head)-1], ',') + + mid, _ := json.Marshal([]interface{}{arr[4], arr[5], arr[6], arr[7], arr[8]}) + // p2 = "," + json(arr[4:9])[1:-1] + "," + p2 := make([]byte, 0, len(mid)+2) + p2 = append(p2, ',') + p2 = append(p2, mid[1:len(mid)-1]...) + p2 = append(p2, ',') + + tail, _ := json.Marshal([]interface{}{ + arr[10], arr[11], arr[12], arr[13], arr[14], arr[15], arr[16], arr[17], + }) + // p3 = "," + json(arr[10:])[1:] => "," + "element1,...,elementN]" + p3 := make([]byte, 0, len(tail)+1) + p3 = append(p3, ',') + p3 = append(p3, tail[1:]...) + + hasher := sha3.New512() + seedB := []byte(seed) + buf := make([]byte, 0, len(p1)+32+len(p2)+16+len(p3)) + b64buf := make([]byte, base64.StdEncoding.EncodedLen(cap(buf))) + + for i := 0; i < maxRequirementsIter; i++ { + d1 := strconv.Itoa(i) + d2 := strconv.Itoa(i >> 1) + + buf = buf[:0] + buf = append(buf, p1...) + buf = append(buf, d1...) + buf = append(buf, p2...) + buf = append(buf, d2...) + buf = append(buf, p3...) + + n := base64.StdEncoding.EncodedLen(len(buf)) + if cap(b64buf) < n { + b64buf = make([]byte, n) + } + b64buf = b64buf[:n] + base64.StdEncoding.Encode(b64buf, buf) -func firstResource(resources []Resources) Resources { - if len(resources) > 0 { - return resources[0] + hasher.Reset() + hasher.Write(seedB) + hasher.Write(b64buf) + sum := hasher.Sum(nil) + + // Python: h[:diff_len] <= target + // diff_len 是字符数(6),target 是字节(3)。Python bytes cmp 按短的逐字节比较。 + // 这里保持等价:取 min(len(target), len(sum)) 字节比较。 + n2 := diffLen + if n2 > len(sum) { + n2 = len(sum) + } + cmpLen := n2 + if cmpLen > len(target) { + cmpLen = len(target) + } + if bytes.Compare(sum[:cmpLen], target[:cmpLen]) <= 0 { + return string(b64buf), true + } } - return Resources{ScriptSources: []string{defaultPowScript}} + return "", false } -func buildConfig(userAgent string, resources Resources) []interface{} { - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - scriptSources := resources.ScriptSources - if len(scriptSources) == 0 { - scriptSources = []string{defaultPowScript} +// SolveProofToken 按服务端挑战求解 proof token(header 用,前缀 gAAAAAB)。 +// 迁移自 gen_image.py.generate_proof_token 的轻量 13 元素 config。 +func SolveProofToken(seed, difficulty, userAgent string) string { + if seed == "" || difficulty == "" { + return "" } - now := time.Now() - perfMs := float64(now.UnixNano()%int64(time.Second)) / float64(time.Millisecond) - return []interface{}{ - screenValues[rng.Intn(len(screenValues))], - legacyParseTime(), - int64(4294705152), - 0, - userAgent, - scriptSources[rng.Intn(len(scriptSources))], - resources.DataBuild, - "en-US", - "en-US,es-US,en,es", - 0, - navigatorKeys[rng.Intn(len(navigatorKeys))], - documentKeys[rng.Intn(len(documentKeys))], - windowKeys[rng.Intn(len(windowKeys))], - perfMs, - uuid.NewString(), - "", - cores[rng.Intn(len(cores))], - float64(now.UnixMilli()) - perfMs, + if userAgent == "" { + userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36 Edg/147.0.0.0" } -} + //nolint:gosec + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + screen := powScreens[rng.Intn(len(powScreens))] * (1 << rng.Intn(3)) // *1/2/4 -func legacyParseTime() string { - loc := time.FixedZone("Eastern Standard Time", -5*60*60) - return time.Now().In(loc).Format("Mon Jan 02 2006 15:04:05") + " GMT-0500 (Eastern Standard Time)" -} + timeStr := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT") -func generate(seed string, difficulty string, config []interface{}) (string, bool) { - target, err := hex.DecodeString(difficulty) - if err != nil || len(target) == 0 { - return "", false + proofConfig := []interface{}{ + screen, // 0 + timeStr, + nil, + 0, // 3 - 迭代 + userAgent, + "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js", + "dpl=1440a687921de39ff5ee56b92807faaadce73f13", + "en", + "en-US", + nil, + "plugins-[object PluginArray]", + powReactListeners[rng.Intn(len(powReactListeners))], + powProofEvents[rng.Intn(len(powProofEvents))], } - diffLen := len(difficulty) / 2 - seedBytes := []byte(seed) - static1 := mustJSONPrefix(config[:3]) - static2 := mustJSONMiddle(config[4:9]) - static3 := mustJSONSuffix(config[10:]) + + diffLen := len(difficulty) hasher := sha3.New512() - for i := 0; i < maxAttempts; i++ { - finalJSON := bytes.NewBuffer(make([]byte, 0, 512)) - finalJSON.Write(static1) - finalJSON.WriteString(fmt.Sprintf("%d", i)) - finalJSON.Write(static2) - finalJSON.WriteString(fmt.Sprintf("%d", i>>1)) - finalJSON.Write(static3) - encoded := []byte(base64.StdEncoding.EncodeToString(finalJSON.Bytes())) - hasher.Write(seedBytes) - hasher.Write(encoded) - digest := hasher.Sum(nil) + for i := 0; i < maxProofIter; i++ { + proofConfig[3] = i + raw, err := json.Marshal(proofConfig) + if err != nil { + continue + } + b64 := base64.StdEncoding.EncodeToString(raw) hasher.Reset() - if bytes.Compare(digest[:diffLen], target) <= 0 { - return string(encoded), true + hasher.Write([]byte(seed + b64)) + sum := hasher.Sum(nil) + hexStr := hex.EncodeToString(sum) + if strings.Compare(hexStr[:diffLen], difficulty) <= 0 { + return powPrefixProof + b64 } } - fallback := "wQ8Lk5FbGpA2NcR9dShT6gYjU7VxZ4D" + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%q", seed))) - return fallback, false -} - -func mustJSONPrefix(values []interface{}) []byte { - b, _ := json.Marshal(values) - return append(b[:len(b)-1], ',') -} - -func mustJSONMiddle(values []interface{}) []byte { - b, _ := json.Marshal(values) - return append(append([]byte{','}, b[1:len(b)-1]...), ',') + return powPrefixProof + powFallback + + base64.StdEncoding.EncodeToString([]byte(`"`+seed+`"`)) } -func mustJSONSuffix(values []interface{}) []byte { - b, _ := json.Marshal(values) - return append([]byte{','}, b[1:]...) +func randomUUID(rng *rand.Rand) string { + var b [16]byte + _, _ = rng.Read(b[:]) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) } // Turnstile solver types diff --git a/util/useragent.go b/util/useragent.go new file mode 100644 index 000000000..d7d561c43 --- /dev/null +++ b/util/useragent.go @@ -0,0 +1,83 @@ +package util + +import ( + "fmt" + "math/rand" + "sync" + "time" +) + +// userAgentSpec 描述一个主流桌面浏览器的 User-Agent 模板 +// 模板中可使用 %d 作为版本占位符 +type userAgentSpec struct { + Template string + MinVersion int + MaxVersion int // 闭区间上界 + Family string +} + +// 模板覆盖 Chrome / Edge / Firefox / Safari 四大主流桌面浏览器, +// 版本号在 [MinVersion, MaxVersion] 闭区间内随机。 +var userAgentSpecs = []userAgentSpec{ + { + Template: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.0.0 Safari/537.36", + MinVersion: 120, + MaxVersion: 147, + Family: "Chrome-Win", + }, + { + Template: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.0.0 Safari/537.36", + MinVersion: 120, + MaxVersion: 147, + Family: "Chrome-Mac", + }, + { + Template: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.0.0 Safari/537.36 Edg/%d.0.0.0", + MinVersion: 120, + MaxVersion: 147, + Family: "Edge-Win", + }, + { + Template: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%d.0) Gecko/20100101 Firefox/%d.0", + MinVersion: 120, + MaxVersion: 132, + Family: "Firefox-Win", + }, + { + Template: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:%d.0) Gecko/20100101 Firefox/%d.0", + MinVersion: 120, + MaxVersion: 132, + Family: "Firefox-Mac", + }, + { + Template: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/%d.0 Safari/605.1.15", + MinVersion: 17, + MaxVersion: 18, + Family: "Safari-Mac", + }, +} + +var ( + uaRand *rand.Rand + uaRandOnce sync.Once +) + +func initUARand() { + uaRandOnce.Do(func() { + uaRand = rand.New(rand.NewSource(time.Now().UnixNano())) + }) +} + +// RandomUserAgent 返回一个随机的主流桌面浏览器 User-Agent +func RandomUserAgent() string { + initUARand() + spec := userAgentSpecs[uaRand.Intn(len(userAgentSpecs))] + + version := spec.MinVersion + if spec.MaxVersion > spec.MinVersion { + version += uaRand.Intn(spec.MaxVersion - spec.MinVersion + 1) + } + + // 部分模板里有两个 %d(如 Edge、Fx),用同一个 version 填充 + return fmt.Sprintf(spec.Template, version, version) +} diff --git a/util/useragent_test.go b/util/useragent_test.go new file mode 100644 index 000000000..010a2829f --- /dev/null +++ b/util/useragent_test.go @@ -0,0 +1,24 @@ +package util + +import ( + "strings" + "testing" +) + +func TestRandomUserAgent(t *testing.T) { + seen := make(map[string]bool) + for i := 0; i < 200; i++ { + ua := RandomUserAgent() + if strings.TrimSpace(ua) == "" { + t.Fatalf("empty UA at iter %d", i) + } + if !strings.HasPrefix(ua, "Mozilla/5.0") { + t.Errorf("UA does not start with Mozilla/5.0: %s", ua) + } + seen[ua] = true + } + // 跑 200 次应至少出现 ≥3 种不同的 UA —— 防止退化到固定值 + if len(seen) < 3 { + t.Errorf("expected at least 3 distinct UAs in 200 draws, got %d", len(seen)) + } +}