commit e72f76267330c39f3998bcd9fcf5c3ced2636c41 Author: lily Date: Fri Jun 5 23:22:37 2026 -0400 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..dae6ef9 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# Endpoints + +``` +GET /autocomplete?term=slow+burn&type=freeform +GET /search?query=enemies+to+lovers&fandom=...&tag=...&rating=...&min_words=...&max_words=...&page=1 +GET /tags/Slow%20Burn/works?page=1 +GET /works/12345 +``` + + - ```PORT``` env var to change the port, defaults to ```8080```. The pattern syntax ```GET /works/{id}``` diff --git a/ao3/ao3.go b/ao3/ao3.go new file mode 100644 index 0000000..a2bcf18 --- /dev/null +++ b/ao3/ao3.go @@ -0,0 +1,406 @@ +package ao3 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "golang.org/x/time/rate" +) + +const ( + baseURL = "https://archiveofourown.org" + userAgent = "ao3tui/0.1 (personal project)" +) + +type Client struct { + http *http.Client + limiter *rate.Limiter +} + +func NewClient() *Client { + jar, _ := cookiejar.New(nil) + return &Client{ + http: &http.Client{ + Jar: jar, + Timeout: 30 * time.Second, + }, + /* + - Rate limit!!!!!!!!!!!!! + */ + limiter: rate.NewLimiter(rate.Every(5*time.Second), 1), + } +} + +func (c *Client) get(ctx context.Context, path string, params url.Values) (*goquery.Document, error) { + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + + u := baseURL + path + if len(params) > 0 { + u += "?" + params.Encode() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == 429 { + return nil, fmt.Errorf("rate limited by AO3") + } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected status: %d", resp.StatusCode) + } + + return goquery.NewDocumentFromReader(resp.Body) +} + +func (c *Client) getJSON(ctx context.Context, path string, params url.Values, out any) error { + if err := c.limiter.Wait(ctx); err != nil { + return err + } + + u := baseURL + path + if len(params) > 0 { + u += "?" + params.Encode() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unexpected status: %d", resp.StatusCode) + } + + return json.NewDecoder(resp.Body).Decode(out) +} + + +type TagSuggestion struct { + Name string + Count int // may be 0 if not returned +} + +type Work struct { + ID int + Title string + Author string + Fandoms []string + Tags []string + Rating string + Warnings []string + Categories []string + WordCount int + ChaptersDone int + ChaptersTotal int // -1 = unknown (?) + Kudos int + Hits int + Summary string + Updated string + URL string +} + +type SearchParams struct { + Query string + Fandom string + Tag string + Rating string // "General Audiences", "Teen And Up Audiences", "Mature", "Explicit", "Not Rated" + MinWords int + MaxWords int + Page int +} + +type SearchResult struct { + Works []Work + TotalWorks int + Page int +} + + +// AutocompleteTags returns tag suggestions for a partial query. +// tagType: "freeform", "fandom", "character", "relationship", "warning" — empty = all +func (c *Client) AutocompleteTags(ctx context.Context, query string, tagType string) ([]TagSuggestion, error) { + params := url.Values{ + "term": {query}, + } + if tagType != "" { + params.Set("type", tagType) + } + + var raw []struct { + ID int `json:"id"` + Name string `json:"name"` + Count int `json:"count"` + } + + if err := c.getJSON(ctx, "/autocomplete/tag", params, &raw); err != nil { + return nil, err + } + + out := make([]TagSuggestion, len(raw)) + for i, r := range raw { + out[i] = TagSuggestion{Name: r.Name, Count: r.Count} + } + return out, nil +} + +func (c *Client) SearchWorks(ctx context.Context, p SearchParams) (SearchResult, error) { + params := url.Values{} + + if p.Query != "" { + params.Set("work_search[query]", p.Query) + } + if p.Fandom != "" { + params.Set("work_search[fandom_names]", p.Fandom) + } + if p.Tag != "" { + params.Set("work_search[other_tag_names]", p.Tag) + } + if p.Rating != "" { + params.Set("work_search[rating_ids]", ratingID(p.Rating)) + } + if p.MinWords > 0 { + params.Set("work_search[words_from]", strconv.Itoa(p.MinWords)) + } + if p.MaxWords > 0 { + params.Set("work_search[words_to]", strconv.Itoa(p.MaxWords)) + } + page := p.Page + if page < 1 { + page = 1 + } + params.Set("page", strconv.Itoa(page)) + + doc, err := c.get(ctx, "/works/search", params) + if err != nil { + return SearchResult{}, err + } + + return parseWorkList(doc, page), nil +} + +// Works by Tag + +func (c *Client) WorksByTag(ctx context.Context, tag string, page int) (SearchResult, error) { + if page < 1 { + page = 1 + } + params := url.Values{"page": {strconv.Itoa(page)}} + encodedTag := url.PathEscape(strings.ReplaceAll(tag, " ", "%20")) + doc, err := c.get(ctx, "/tags/"+encodedTag+"/works", params) + if err != nil { + return SearchResult{}, err + } + return parseWorkList(doc, page), nil +} + +// Work Detail + +func (c *Client) GetWork(ctx context.Context, id int) (Work, error) { + params := url.Values{"view_adult": {"true"}} + doc, err := c.get(ctx, fmt.Sprintf("/works/%d", id), params) + if err != nil { + return Work{}, err + } + return parseWorkDetail(doc, id), nil +} + +// Parsers + +func parseWorkList(doc *goquery.Document, page int) SearchResult { + var result SearchResult + result.Page = page + + /* + // total count — format is "223,747 Found" + */ + doc.Find("h3.heading").Each(func(_ int, s *goquery.Selection) { + text := strings.TrimSpace(s.Text()) + if strings.Contains(text, "Found") { + raw := strings.Split(text, " ")[0] + raw = strings.ReplaceAll(raw, ",", "") + result.TotalWorks, _ = strconv.Atoi(raw) + } + }) + + doc.Find("li.work.blurb.group").Each(func(_ int, s *goquery.Selection) { + w := parseWorkBlurb(s) + result.Works = append(result.Works, w) + }) + + return result +} + +func parseWorkBlurb(s *goquery.Selection) Work { + var w Work + + // ID from href + if href, exists := s.Find("h4.heading a").First().Attr("href"); exists { + fmt.Sscanf(href, "/works/%d", &w.ID) + w.URL = baseURL + href + } + + w.Title = strings.TrimSpace(s.Find("h4.heading a").First().Text()) + + // Authors (may be "Anonymous") + s.Find("h4.heading a[rel='author']").Each(func(_ int, a *goquery.Selection) { + w.Author = strings.TrimSpace(a.Text()) + }) + if w.Author == "" { + w.Author = "Anonymous" + } + + // Fandoms + s.Find("h5.fandoms a.tag").Each(func(_ int, a *goquery.Selection) { + w.Fandoms = append(w.Fandoms, strings.TrimSpace(a.Text())) + }) + + // Tags + s.Find("ul.tags li.freeforms a.tag").Each(func(_ int, a *goquery.Selection) { + w.Tags = append(w.Tags, strings.TrimSpace(a.Text())) + }) + + // Rating — class is like "rating-mature rating" + s.Find("ul.required-tags span[class*='rating-']").Each(func(_ int, span *goquery.Selection) { + if t := span.AttrOr("title", ""); t != "" { + w.Rating = t + } + }) + + // Categories — class is like "category-slash category" + s.Find("ul.required-tags span[class*='category-']").Each(func(_ int, span *goquery.Selection) { + if t := span.AttrOr("title", ""); t != "" { + w.Categories = append(w.Categories, t) + } + }) + + // Stats + w.WordCount = parseStatInt(s, "dd.words") + w.Kudos = parseStatInt(s, "dd.kudos") + w.Hits = parseStatInt(s, "dd.hits") + + // Chapters + chapText := strings.TrimSpace(s.Find("dd.chapters").Text()) + if chapText != "" { + parts := strings.Split(chapText, "/") + if len(parts) == 2 { + fmt.Sscanf(parts[0], "%d", &w.ChaptersDone) + if parts[1] == "?" { + w.ChaptersTotal = -1 + } else { + fmt.Sscanf(parts[1], "%d", &w.ChaptersTotal) + } + } + } + + // Summary + w.Summary = strings.TrimSpace(s.Find("blockquote.userstuff.summary").Text()) + + // Updated date + w.Updated = strings.TrimSpace(s.Find("p.datetime").Text()) + + return w +} + +func parseWorkDetail(doc *goquery.Document, id int) Work { + var w Work + w.ID = id + w.URL = fmt.Sprintf("%s/works/%d", baseURL, id) + + w.Title = strings.TrimSpace(doc.Find("h2.title.heading").First().Text()) + + doc.Find("h3.byline a[rel='author']").Each(func(_ int, a *goquery.Selection) { + w.Author = strings.TrimSpace(a.Text()) + }) + if w.Author == "" { + w.Author = "Anonymous" + } + + doc.Find("dd.fandom a.tag").Each(func(_ int, a *goquery.Selection) { + w.Fandoms = append(w.Fandoms, strings.TrimSpace(a.Text())) + }) + + doc.Find("dd.freeform a.tag").Each(func(_ int, a *goquery.Selection) { + w.Tags = append(w.Tags, strings.TrimSpace(a.Text())) + }) + + w.Rating = strings.TrimSpace(doc.Find("dd.rating a.tag").Text()) + + doc.Find("dd.warning a.tag").Each(func(_ int, a *goquery.Selection) { + w.Warnings = append(w.Warnings, strings.TrimSpace(a.Text())) + }) + + doc.Find("dd.category a.tag").Each(func(_ int, a *goquery.Selection) { + w.Categories = append(w.Categories, strings.TrimSpace(a.Text())) + }) + + w.WordCount = parseStatInt(doc.Selection, "dd.words") + w.Kudos = parseStatInt(doc.Selection, "dd.kudos") + w.Hits = parseStatInt(doc.Selection, "dd.hits") + + chapText := strings.TrimSpace(doc.Find("dd.chapters").Text()) + if chapText != "" { + parts := strings.Split(chapText, "/") + if len(parts) == 2 { + fmt.Sscanf(parts[0], "%d", &w.ChaptersDone) + if parts[1] == "?" { + w.ChaptersTotal = -1 + } else { + fmt.Sscanf(parts[1], "%d", &w.ChaptersTotal) + } + } + } + + w.Summary = strings.TrimSpace(doc.Find("div.summary blockquote.userstuff").Text()) + w.Updated = strings.TrimSpace(doc.Find("dd.status").Text()) + + return w +} + +func parseStatInt(s *goquery.Selection, selector string) int { + text := strings.ReplaceAll(strings.TrimSpace(s.Find(selector).Text()), ",", "") + n, _ := strconv.Atoi(text) + return n +} + +func ratingID(rating string) string { + switch strings.ToLower(rating) { + case "general audiences", "general", "g": + return "10" + case "teen and up audiences", "teen", "t": + return "11" + case "mature", "m": + return "12" + case "explicit", "e": + return "13" + default: + return "9" // Not Rated + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c1ec444 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module ao3scraper + +go 1.21 + +require ( + github.com/PuerkitoBio/goquery v1.9.2 + golang.org/x/time v0.5.0 +) + +require ( + github.com/andybalholm/cascadia v1.3.2 // indirect + golang.org/x/net v0.24.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1680545 --- /dev/null +++ b/go.sum @@ -0,0 +1,42 @@ +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..2da9b79 --- /dev/null +++ b/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "strconv" + + "ao3scraper/ao3" +) + +var client *ao3.Client + +func main() { + client = ao3.NewClient() + + port := os.Getenv("PORT") + if port == "" { + port = "8080" + } + + mux := http.NewServeMux() + mux.HandleFunc("GET /autocomplete", handleAutocomplete) + mux.HandleFunc("GET /search", handleSearch) + mux.HandleFunc("GET /tags/{tag}/works", handleTagWorks) + mux.HandleFunc("GET /works/{id}", handleWork) + + log.Printf("listening on :%s", port) + log.Fatal(http.ListenAndServe(":"+port, mux)) +} + +func handleAutocomplete(w http.ResponseWriter, r *http.Request) { + term := r.URL.Query().Get("term") + tagType := r.URL.Query().Get("type") + if term == "" { + jsonError(w, "term required", http.StatusBadRequest) + return + } + + tags, err := client.AutocompleteTags(r.Context(), term, tagType) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonOK(w, tags) +} + +func handleSearch(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + page, _ := strconv.Atoi(q.Get("page")) + minWords, _ := strconv.Atoi(q.Get("min_words")) + maxWords, _ := strconv.Atoi(q.Get("max_words")) + + result, err := client.SearchWorks(r.Context(), ao3.SearchParams{ + Query: q.Get("query"), + Fandom: q.Get("fandom"), + Tag: q.Get("tag"), + Rating: q.Get("rating"), + MinWords: minWords, + MaxWords: maxWords, + Page: page, + }) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonOK(w, result) +} + +func handleTagWorks(w http.ResponseWriter, r *http.Request) { + tag := r.PathValue("tag") + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + + result, err := client.WorksByTag(r.Context(), tag, page) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonOK(w, result) +} + +func handleWork(w http.ResponseWriter, r *http.Request) { + id, err := strconv.Atoi(r.PathValue("id")) + if err != nil { + jsonError(w, "invalid id", http.StatusBadRequest) + return + } + + work, err := client.GetWork(r.Context(), id) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonOK(w, work) +} + +func jsonOK(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func jsonError(w http.ResponseWriter, msg string, code int) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + json.NewEncoder(w).Encode(map[string]string{"error": msg}) +}