push to remote

mangacrawler/chapterdownload.go

@@ -0,0 +1,77 @@
+package mangacrawler
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+)
+
+type Chapter struct {
+	Url  string      `json:"baseUrl"`
+	Data ChapterData `json:"chapter"`
+}
+
+type ChapterData struct {
+	Pages []string `json:"data"`
+	Hash  string   `json:"hash"`
+}
+
+func chapterDownload(chapterId string, chapterPath string, chapterNo string) {
+	var pages Chapter
+
+	url := "https://api.mangadex.org/at-home/server/" + chapterId
+	data := GetJson(url)
+
+	if err := json.Unmarshal(data, &pages); err != nil {
+		log.Fatal(err)
+	}
+
+	for _, page := range pages.Data.Pages {
+		url = strings.Join([]string{pages.Url, "data", pages.Data.Hash, page}, "/")
+		pageDownload(url, chapterPath, page, chapterNo)
+	}
+}
+
+func pageDownload(url string, path string, page string, chapterNo string) {
+	filepage := page
+	regMatch, _ := regexp.MatchString(`^\D`, filepage)
+	if regMatch {
+		filepage = filepage[1:]
+	}
+	fileSplit := strings.Split(filepage, ".")
+	filepage = strings.Join([]string{fmt.Sprintf("%067s", fileSplit[0]), fileSplit[1]}, ".")
+
+	if _, err := os.Stat(path + "/chapter" + chapterNo + "_" + filepage); err == nil {
+		return
+	}
+
+	result, err := http.Get(url)
+	if err != nil {
+		panic(err)
+	}
+	defer result.Body.Close()
+
+	if result.StatusCode != 200 {
+		pageDownload(url, path, page, chapterNo)
+		return
+	}
+
+	file, err := os.Create(path + "/chapter" + chapterNo + "_" + filepage)
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = io.Copy(file, result.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	file.Close()
+
+	fmt.Printf("Downloading: %s\n", filepage)
+}

mangacrawler/chaptersearch.go

@@ -0,0 +1,74 @@
+package mangacrawler
+
+import (
+	"encoding/json"
+	"strconv"
+)
+
+type Chapters struct {
+	Data  []ChaptersData `json:"data"`
+	Total int            `json:"total"`
+}
+
+type ChaptersData struct {
+	Id         string             `json:"id"`
+	Attributes ChaptersAttributes `json:"attributes"`
+	Rels       []ChaptersRels     `json:"relationships"`
+}
+
+type ChaptersAttributes struct {
+	Volume   string `json:"volume"`
+	Chapter  string `json:"chapter"`
+	Title    string `json:"title"`
+	Language string `json:"translatedLanguage"`
+}
+
+type ChaptersRels struct {
+	RelsAttr RelsAttributes `json:"attributes"`
+}
+
+type RelsAttributes struct {
+	Name string `json:"name"`
+}
+
+func getChapterInfo(mangaId string) []ChaptersData {
+	var tempChapters Chapters
+	var chapters Chapters
+
+	url := "https://api.mangadex.org/manga/" + mangaId + "/feed"
+
+	data := GetJson(url)
+
+	if err := json.Unmarshal(data, &tempChapters); err != nil {
+		panic(err)
+	}
+
+	if tempChapters.Total > 100 {
+		var chaptersOffset Chapters
+		offset := 1
+		maxOffset := tempChapters.Total / 100
+
+		for offset <= maxOffset {
+			url = "https://api.mangadex.org/manga/" + mangaId + "/feed?offset=" + strconv.Itoa(offset*100)
+
+			data = GetJson(url)
+
+			if err := json.Unmarshal(data, &chaptersOffset); err != nil {
+				panic(err)
+			}
+
+			tempChapters.Data = append(tempChapters.Data, chaptersOffset.Data...)
+
+			offset++
+		}
+	}
+
+	for _, chapter := range tempChapters.Data {
+		if chapter.Attributes.Language == "en" {
+			chapters.Data = append(chapters.Data, chapter)
+		}
+	}
+
+	return chapters.Data
+
+}

mangacrawler/mangacrawler.go

@@ -0,0 +1,74 @@
+package mangacrawler
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type MangaYaml struct {
+	Name      string
+	ID        string
+	Chapter   float64
+	Completed bool
+}
+
+func GetManga(manga MangaYaml, filepath string, forceDl bool) (MangaYaml, bool) {
+	chaptersData := getChapterInfo(manga.ID)
+	newChapter := false
+	latestChapter := manga.Chapter
+
+	// set subdirs for chapters in style volume-chapter-name
+	for _, chapter := range chaptersData {
+		// chapterVolume := chapter.Attributes.Volume
+		chapterIndex, _ := strconv.ParseFloat(chapter.Attributes.Chapter, 32)
+		if chapterIndex > manga.Chapter || forceDl {
+			newChapter = true
+			chapterChapter := fmt.Sprintf("%03s", chapter.Attributes.Chapter)
+			extraChapter := strings.Split(chapterChapter, ".")
+			if len(extraChapter) > 1 {
+				chapterChapter = strings.Join([]string{fmt.Sprintf("%03s", extraChapter[0]), "z" + extraChapter[1]}, "-")
+			}
+			chapterTitle := chapter.Attributes.Title
+
+			fmt.Printf("Working on Chapter: %s %s\n", chapterChapter, chapterTitle)
+			chapterpath := strings.Join([]string{filepath, "chapter" + chapterChapter}, "/")
+			if len(chapterTitle) > 0 {
+				chapterpath = strings.Join([]string{filepath, "chapter" + chapterChapter + "-" + chapterTitle}, "/")
+			}
+			os.MkdirAll(chapterpath, 0755)
+			chapterDownload(chapter.Id, chapterpath, chapterChapter)
+			fmt.Println()
+			time.Sleep(1 * time.Second)
+		}
+		if chapterIndex > latestChapter {
+			latestChapter = chapterIndex
+		}
+	}
+	if !newChapter {
+		fmt.Print("  No new chapter released yet!\n\n")
+	}
+
+	manga.Chapter = latestChapter
+
+	return manga, newChapter
+}
+
+func GetJson(url string) []byte {
+	response, err := http.Get(url)
+	if err != nil {
+		panic(err)
+	}
+	defer response.Body.Close()
+
+	data, err := io.ReadAll(response.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	return data
+}

mangacrawler/mangasearch.go

@@ -0,0 +1,76 @@
+package mangacrawler
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type Manga struct {
+	Data MangaData `json:"data"`
+}
+
+type MangaData struct {
+	Attributes MangaAttributes `json:"attributes"`
+}
+
+type MangaAttributes struct {
+	Title       Titles   `json:"title"`
+	AltTitle    []Titles `json:"altTitles"`
+	Status      string   `json:"status"`
+	LastChapter string   `json:"lastChapter"`
+}
+
+type Titles struct {
+	JP string `json:"ja-ro"`
+	EN string `json:"en"`
+}
+
+func GetMangaInfo(mangaYaml MangaYaml) (string, bool) {
+	var manga Manga
+	status := false
+	homepath, _ := os.UserHomeDir()
+
+	url := "https://api.mangadex.org/manga/" + mangaYaml.ID
+	data := GetJson(url)
+	if err := json.Unmarshal(data, &manga); err != nil {
+		panic(err)
+	}
+
+	mangaLastChapter, _ := strconv.ParseFloat(manga.Data.Attributes.LastChapter, 32)
+	if manga.Data.Attributes.Status == "completed" && (mangaLastChapter <= mangaYaml.Chapter || manga.Data.Attributes.LastChapter == "") {
+		status = true
+	}
+
+	// set home directory and create subdir to save manga in
+	mangaTitles := []string{manga.Data.Attributes.Title.EN}
+	for _, title := range manga.Data.Attributes.AltTitle {
+		if title.EN != "" {
+			mangaTitles = append(mangaTitles, title.EN)
+		} else if title.JP != "" {
+			mangaTitles = append(mangaTitles, title.JP)
+		}
+	}
+	for _, title := range mangaTitles {
+		if _, err := os.Stat(strings.Join([]string{homepath, "mangas/MangaDex", title}, "/")); err == nil && title != "" {
+			fmt.Printf("Title found on system! Using: %s\n", title)
+			return title, status
+		}
+	}
+
+	for i, title := range mangaTitles {
+		fmt.Printf("(%d): %s\n", i, title)
+	}
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Print("---\nPlease choose title for the manga: ")
+	selection, _ := reader.ReadString('\n')
+	selection = strings.TrimSuffix(selection, "\n")
+	choice, _ := strconv.Atoi(selection)
+
+	mangaTitle := mangaTitles[choice]
+
+	return mangaTitle, status
+}