From 62fb8b1795996749ce6bdd5c2e84ec9a4bc4de50 Mon Sep 17 00:00:00 2001 From: hiddifydeveloper Date: Sun, 26 Jul 2026 00:11:01 +0330 Subject: [PATCH] ezytel: add RPC tests, unified message format, translation-provider seam - GetChannelMessages/GetChannelInfo tests covering structured parsing, stale-cache fallback, and inline-image behavior. - New ChannelMessage proto message + parseMessages() so callers get a structured post list instead of re-scraping the raw HTML blob. - Extracted the Google Translate domain-fronting logic into a frontProvider interface (front.go) so a future provider can be swapped in without touching the parsing code. --- v2/ezytel/ezytel.pb.go | 247 ++++++++++++++++++++++++------- v2/ezytel/ezytel.proto | 22 +++ v2/ezytel/ezytel_service_imp.go | 253 ++++++++++++++------------------ v2/ezytel/ezytel_test.go | 142 +++++++++++++++++- v2/ezytel/front.go | 166 +++++++++++++++++++++ 5 files changed, 625 insertions(+), 205 deletions(-) create mode 100644 v2/ezytel/front.go diff --git a/v2/ezytel/ezytel.pb.go b/v2/ezytel/ezytel.pb.go index de3c61ac0..6a8ed9e67 100644 --- a/v2/ezytel/ezytel.pb.go +++ b/v2/ezytel/ezytel.pb.go @@ -295,6 +295,9 @@ type ChannelMessagesResponse struct { ChannelAvatar string `protobuf:"bytes,2,opt,name=channel_avatar,json=channelAvatar,proto3" json:"channel_avatar,omitempty"` // Last post id seen on the page (mirrors the lastread_ cookie). LastPostId int64 `protobuf:"varint,3,opt,name=last_post_id,json=lastPostId,proto3" json:"last_post_id,omitempty"` + // Structured parse of the same page, one entry per post, in the + // same order they appear in html. Prefer this over scraping html. + Messages []*ChannelMessage `protobuf:"bytes,4,rep,name=messages,proto3" json:"messages,omitempty"` } func (x *ChannelMessagesResponse) Reset() { @@ -350,6 +353,110 @@ func (x *ChannelMessagesResponse) GetLastPostId() int64 { return 0 } +func (x *ChannelMessagesResponse) GetMessages() []*ChannelMessage { + if x != nil { + return x.Messages + } + return nil +} + +// A single Telegram post, parsed out of the channel's public HTML page. +type ChannelMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Post id (the numeric suffix of data-post="/"). + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Plain-text body with HTML tags stripped. Empty for media-only posts. + Text string `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` + // Image URLs referenced by the post, already rewritten to + // "proxy.php?url=" (or a "data:image/jpeg;base64,…" URI when + // the request did not set disable_inline_images). + MediaUrls []string `protobuf:"bytes,3,rep,name=media_urls,json=mediaUrls,proto3" json:"media_urls,omitempty"` + // Unix epoch seconds of the post. + Date int64 `protobuf:"varint,4,opt,name=date,proto3" json:"date,omitempty"` + // Persian-calendar formatted date/time string, same convention as + // ChannelInfo.date_str. + DateStr string `protobuf:"bytes,5,opt,name=date_str,json=dateStr,proto3" json:"date_str,omitempty"` + // True if the post is a forward from another channel. + IsForward bool `protobuf:"varint,6,opt,name=is_forward,json=isForward,proto3" json:"is_forward,omitempty"` +} + +func (x *ChannelMessage) Reset() { + *x = ChannelMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_v2_ezytel_ezytel_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChannelMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelMessage) ProtoMessage() {} + +func (x *ChannelMessage) ProtoReflect() protoreflect.Message { + mi := &file_v2_ezytel_ezytel_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelMessage.ProtoReflect.Descriptor instead. +func (*ChannelMessage) Descriptor() ([]byte, []int) { + return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{4} +} + +func (x *ChannelMessage) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ChannelMessage) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +func (x *ChannelMessage) GetMediaUrls() []string { + if x != nil { + return x.MediaUrls + } + return nil +} + +func (x *ChannelMessage) GetDate() int64 { + if x != nil { + return x.Date + } + return 0 +} + +func (x *ChannelMessage) GetDateStr() string { + if x != nil { + return x.DateStr + } + return "" +} + +func (x *ChannelMessage) GetIsForward() bool { + if x != nil { + return x.IsForward + } + return false +} + // Fetch a Telegram-hosted image through the translate.goog domain front // and cache it locally. Mirrors proxy.php?url=. type ProxyImageRequest struct { @@ -365,7 +472,7 @@ type ProxyImageRequest struct { func (x *ProxyImageRequest) Reset() { *x = ProxyImageRequest{} if protoimpl.UnsafeEnabled { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[4] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -378,7 +485,7 @@ func (x *ProxyImageRequest) String() string { func (*ProxyImageRequest) ProtoMessage() {} func (x *ProxyImageRequest) ProtoReflect() protoreflect.Message { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[4] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -391,7 +498,7 @@ func (x *ProxyImageRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyImageRequest.ProtoReflect.Descriptor instead. func (*ProxyImageRequest) Descriptor() ([]byte, []int) { - return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{4} + return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{5} } func (x *ProxyImageRequest) GetHexUrl() string { @@ -416,7 +523,7 @@ type ProxyImageResponse struct { func (x *ProxyImageResponse) Reset() { *x = ProxyImageResponse{} if protoimpl.UnsafeEnabled { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[5] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -429,7 +536,7 @@ func (x *ProxyImageResponse) String() string { func (*ProxyImageResponse) ProtoMessage() {} func (x *ProxyImageResponse) ProtoReflect() protoreflect.Message { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[5] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -442,7 +549,7 @@ func (x *ProxyImageResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyImageResponse.ProtoReflect.Descriptor instead. func (*ProxyImageResponse) Descriptor() ([]byte, []int) { - return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{5} + return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{6} } func (x *ProxyImageResponse) GetData() []byte { @@ -479,7 +586,7 @@ type ParseChannelsRequest struct { func (x *ParseChannelsRequest) Reset() { *x = ParseChannelsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[6] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -492,7 +599,7 @@ func (x *ParseChannelsRequest) String() string { func (*ParseChannelsRequest) ProtoMessage() {} func (x *ParseChannelsRequest) ProtoReflect() protoreflect.Message { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[6] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -505,7 +612,7 @@ func (x *ParseChannelsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ParseChannelsRequest.ProtoReflect.Descriptor instead. func (*ParseChannelsRequest) Descriptor() ([]byte, []int) { - return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{6} + return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{7} } func (x *ParseChannelsRequest) GetRaw() string { @@ -526,7 +633,7 @@ type ParseChannelsResponse struct { func (x *ParseChannelsResponse) Reset() { *x = ParseChannelsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[7] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -539,7 +646,7 @@ func (x *ParseChannelsResponse) String() string { func (*ParseChannelsResponse) ProtoMessage() {} func (x *ParseChannelsResponse) ProtoReflect() protoreflect.Message { - mi := &file_v2_ezytel_ezytel_proto_msgTypes[7] + mi := &file_v2_ezytel_ezytel_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -552,7 +659,7 @@ func (x *ParseChannelsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ParseChannelsResponse.ProtoReflect.Descriptor instead. func (*ParseChannelsResponse) Descriptor() ([]byte, []int) { - return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{7} + return file_v2_ezytel_ezytel_proto_rawDescGZIP(), []int{8} } func (x *ParseChannelsResponse) GetChannelIds() []string { @@ -597,36 +704,50 @@ var file_v2_ezytel_ezytel_proto_rawDesc = []byte{ 0x03, 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, - 0x65, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x22, 0x76, 0x0a, - 0x17, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x74, 0x6d, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x74, 0x6d, 0x6c, 0x12, 0x25, 0x0a, 0x0e, - 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x76, 0x61, - 0x74, 0x61, 0x72, 0x12, 0x20, 0x0a, 0x0c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x6f, 0x73, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x50, - 0x6f, 0x73, 0x74, 0x49, 0x64, 0x22, 0x2c, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6d, - 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x65, - 0x78, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x68, 0x65, 0x78, - 0x55, 0x72, 0x6c, 0x22, 0x6a, 0x0a, 0x12, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6d, 0x61, 0x67, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, - 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0x28, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x73, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x61, 0x77, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x61, 0x77, 0x22, 0x38, 0x0a, 0x15, 0x50, 0x61, 0x72, - 0x73, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, - 0x49, 0x64, 0x73, 0x42, 0x4b, 0x0a, 0x1e, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x69, 0x64, 0x64, 0x69, - 0x66, 0x79, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x32, 0x2e, 0x65, - 0x7a, 0x79, 0x74, 0x65, 0x6c, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x68, 0x69, 0x64, 0x64, 0x69, 0x66, 0x79, 0x2f, 0x68, 0x69, 0x64, 0x64, 0x69, 0x66, - 0x79, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x32, 0x2f, 0x65, 0x7a, 0x79, 0x74, 0x65, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x22, 0xaa, 0x01, + 0x0a, 0x17, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x74, 0x6d, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x74, 0x6d, 0x6c, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x76, 0x61, 0x74, 0x61, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x76, + 0x61, 0x74, 0x61, 0x72, 0x12, 0x20, 0x0a, 0x0c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x6f, 0x73, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, + 0x50, 0x6f, 0x73, 0x74, 0x49, 0x64, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x65, 0x7a, 0x79, 0x74, 0x65, + 0x6c, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x52, 0x08, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0xa1, 0x01, 0x0a, 0x0e, 0x43, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, + 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x55, 0x72, 0x6c, 0x73, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x72, 0x12, + 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x22, 0x2c, + 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x68, 0x65, 0x78, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x68, 0x65, 0x78, 0x55, 0x72, 0x6c, 0x22, 0x6a, 0x0a, 0x12, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x61, 0x63, + 0x68, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x61, 0x63, 0x68, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x28, 0x0a, 0x14, 0x50, 0x61, 0x72, 0x73, + 0x65, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x10, 0x0a, 0x03, 0x72, 0x61, 0x77, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, + 0x61, 0x77, 0x22, 0x38, 0x0a, 0x15, 0x50, 0x61, 0x72, 0x73, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x73, 0x42, 0x4b, 0x0a, 0x1e, + 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x69, 0x64, 0x64, 0x69, 0x66, 0x79, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x32, 0x2e, 0x65, 0x7a, 0x79, 0x74, 0x65, 0x6c, 0x5a, 0x29, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x69, 0x64, 0x64, 0x69, + 0x66, 0x79, 0x2f, 0x68, 0x69, 0x64, 0x64, 0x69, 0x66, 0x79, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x2f, + 0x76, 0x32, 0x2f, 0x65, 0x7a, 0x79, 0x74, 0x65, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -641,23 +762,25 @@ func file_v2_ezytel_ezytel_proto_rawDescGZIP() []byte { return file_v2_ezytel_ezytel_proto_rawDescData } -var file_v2_ezytel_ezytel_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_v2_ezytel_ezytel_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_v2_ezytel_ezytel_proto_goTypes = []any{ (*ChannelInfoRequest)(nil), // 0: ezytel.ChannelInfoRequest (*ChannelInfo)(nil), // 1: ezytel.ChannelInfo (*ChannelMessagesRequest)(nil), // 2: ezytel.ChannelMessagesRequest (*ChannelMessagesResponse)(nil), // 3: ezytel.ChannelMessagesResponse - (*ProxyImageRequest)(nil), // 4: ezytel.ProxyImageRequest - (*ProxyImageResponse)(nil), // 5: ezytel.ProxyImageResponse - (*ParseChannelsRequest)(nil), // 6: ezytel.ParseChannelsRequest - (*ParseChannelsResponse)(nil), // 7: ezytel.ParseChannelsResponse + (*ChannelMessage)(nil), // 4: ezytel.ChannelMessage + (*ProxyImageRequest)(nil), // 5: ezytel.ProxyImageRequest + (*ProxyImageResponse)(nil), // 6: ezytel.ProxyImageResponse + (*ParseChannelsRequest)(nil), // 7: ezytel.ParseChannelsRequest + (*ParseChannelsResponse)(nil), // 8: ezytel.ParseChannelsResponse } var file_v2_ezytel_ezytel_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 4, // 0: ezytel.ChannelMessagesResponse.messages:type_name -> ezytel.ChannelMessage + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_v2_ezytel_ezytel_proto_init() } @@ -715,7 +838,7 @@ func file_v2_ezytel_ezytel_proto_init() { } } file_v2_ezytel_ezytel_proto_msgTypes[4].Exporter = func(v any, i int) any { - switch v := v.(*ProxyImageRequest); i { + switch v := v.(*ChannelMessage); i { case 0: return &v.state case 1: @@ -727,7 +850,7 @@ func file_v2_ezytel_ezytel_proto_init() { } } file_v2_ezytel_ezytel_proto_msgTypes[5].Exporter = func(v any, i int) any { - switch v := v.(*ProxyImageResponse); i { + switch v := v.(*ProxyImageRequest); i { case 0: return &v.state case 1: @@ -739,7 +862,7 @@ func file_v2_ezytel_ezytel_proto_init() { } } file_v2_ezytel_ezytel_proto_msgTypes[6].Exporter = func(v any, i int) any { - switch v := v.(*ParseChannelsRequest); i { + switch v := v.(*ProxyImageResponse); i { case 0: return &v.state case 1: @@ -751,6 +874,18 @@ func file_v2_ezytel_ezytel_proto_init() { } } file_v2_ezytel_ezytel_proto_msgTypes[7].Exporter = func(v any, i int) any { + switch v := v.(*ParseChannelsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v2_ezytel_ezytel_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*ParseChannelsResponse); i { case 0: return &v.state @@ -769,7 +904,7 @@ func file_v2_ezytel_ezytel_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v2_ezytel_ezytel_proto_rawDesc, NumEnums: 0, - NumMessages: 8, + NumMessages: 9, NumExtensions: 0, NumServices: 0, }, diff --git a/v2/ezytel/ezytel.proto b/v2/ezytel/ezytel.proto index 21a68ac40..19f904f75 100644 --- a/v2/ezytel/ezytel.proto +++ b/v2/ezytel/ezytel.proto @@ -69,6 +69,28 @@ message ChannelMessagesResponse { string channel_avatar = 2; // Last post id seen on the page (mirrors the lastread_ cookie). int64 last_post_id = 3; + // Structured parse of the same page, one entry per post, in the + // same order they appear in html. Prefer this over scraping html. + repeated ChannelMessage messages = 4; +} + +// A single Telegram post, parsed out of the channel's public HTML page. +message ChannelMessage { + // Post id (the numeric suffix of data-post="/"). + string id = 1; + // Plain-text body with HTML tags stripped. Empty for media-only posts. + string text = 2; + // Image URLs referenced by the post, already rewritten to + // "proxy.php?url=" (or a "data:image/jpeg;base64,…" URI when + // the request did not set disable_inline_images). + repeated string media_urls = 3; + // Unix epoch seconds of the post. + int64 date = 4; + // Persian-calendar formatted date/time string, same convention as + // ChannelInfo.date_str. + string date_str = 5; + // True if the post is a forward from another channel. + bool is_forward = 6; } // Fetch a Telegram-hosted image through the translate.goog domain front diff --git a/v2/ezytel/ezytel_service_imp.go b/v2/ezytel/ezytel_service_imp.go index 4438b5351..050427347 100644 --- a/v2/ezytel/ezytel_service_imp.go +++ b/v2/ezytel/ezytel_service_imp.go @@ -8,10 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io" - "math/rand" "net/http" - "net/url" "os" "path/filepath" "regexp" @@ -30,6 +27,7 @@ type EzytelService struct { cacheDir string client *http.Client + front frontProvider mu sync.Mutex infoMu sync.Mutex } @@ -40,15 +38,17 @@ func NewEzytelService(cacheDir string) *EzytelService { if cacheDir == "" { cacheDir = filepath.Join(os.TempDir(), "ezytel-cache") } + client := &http.Client{ + Timeout: 35 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + ResponseHeaderTimeout: 30 * time.Second, + }, + } return &EzytelService{ cacheDir: cacheDir, - client: &http.Client{ - Timeout: 35 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - ResponseHeaderTimeout: 30 * time.Second, - }, - }, + client: client, + front: &googleTranslateFront{client: client}, } } @@ -76,7 +76,7 @@ func (s *EzytelService) GetChannelMessages(ctx context.Context, in *ChannelMessa if in.Before != 0 { path = fmt.Sprintf("%s?before=%d", chid, in.Before) } - html, err := s.curlAuto(ctx, path) + html, err := s.front.FetchPage(ctx, path) if err != nil { return nil, err } @@ -89,6 +89,13 @@ func (s *EzytelService) GetChannelMessages(ctx context.Context, in *ChannelMessa html = rewriteBackgroundImages(html) // rewrite html = rewriteImgSources(html) + + // Capture the main content with its original