diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b326602 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module imagecrawler + +go 1.24.3 diff --git a/imagecrawler b/imagecrawler new file mode 100755 index 0000000..8f05024 Binary files /dev/null and b/imagecrawler differ diff --git a/imagecrawler.go b/imagecrawler.go index 2f78671..fcfadfd 100755 --- a/imagecrawler.go +++ b/imagecrawler.go @@ -9,7 +9,6 @@ import ( "net/http" "net/url" "os" - "os/user" "strconv" "strings" "sync" @@ -44,7 +43,7 @@ func main() { page := 1 ratio := 0.0 - flag.StringVar(&path, "dir", "unnamed", "Directory to safe pictures. Default is %HOME/pictures/konachan/unnamed") + flag.StringVar(&path, "dir", "", "Path to safe pictures. Default is '~/pictures/$site/$tags'") flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false") flag.StringVar(&tags, "tags", "", "Tags used to filter search query.") flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have") @@ -52,19 +51,27 @@ func main() { flag.Parse() - // set home directory and create it to save pictures in - homepath, err := user.Current() + // get the UserHomeDir + homedir, err := os.UserHomeDir() if err != nil { log.Fatal(err) } - filepath := strings.Join([]string{homepath.HomeDir, "pictures", site, strings.TrimSuffix(path, "\n")}, "/") - os.MkdirAll(filepath, 0700) // edit tags array to met API requirement + tags = strings.Replace(tags, " ", "_", -1) tags = strings.Replace(tags, ",", "+", -1) tags = strings.Replace(tags, "=", ":", -1) tags = strings.TrimSuffix(tags, "\n") + // set the path based on variable or tags + default_path := strings.Join([]string{homedir, "pictures", site, tags}, "/") + filepath := default_path + if path != "" { + filepath = path + } + + os.MkdirAll(filepath, 0644) + // calculate aspect ratio if isFlagPassed("aspect") { aspectSlice := strings.Split(aspect, ":") @@ -79,9 +86,9 @@ func main() { fmt.Println("Page: ", page) - website := fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s", page, tags) + website := fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s", page, tags) if safemode { - website = fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s+rating:safe", page, tags) + website = fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s+rating:safe", page, tags) } if site == "danbooru" { @@ -91,7 +98,16 @@ func main() { } } - picList := openConnection(website) + response := requestAPI(website) + fmt.Println(response.StatusCode) + if response.StatusCode == 421 { + time.Sleep(1 * time.Second) + } + if response.StatusCode != 200 { + defer response.Body.Close() + continue + } + picList := getJSON(response) pictures, count := parseMaps(picList, ratio) picHits = count @@ -117,15 +133,19 @@ func isFlagPassed(name string) bool { } // function to create the connection to konachan and get the API response -func openConnection(url string) []picture { - var f []picture - +func requestAPI(url string) *http.Response { result, err := http.Get(url) if err != nil { log.Fatal(err) } - defer result.Body.Close() + return result +} + +func getJSON(result *http.Response) []picture { + var f []picture + + defer result.Body.Close() data, err := io.ReadAll(result.Body) if err != nil { log.Fatal(err)