change default save location and add go mod

This commit is contained in:
m3philis
2025-06-01 16:32:48 +02:00
parent 693290bc03
commit 00d25bbe3a
3 changed files with 36 additions and 13 deletions

3
go.mod Normal file
View File

@@ -0,0 +1,3 @@
module imagecrawler
go 1.24.3

BIN
imagecrawler Executable file

Binary file not shown.

View File

@@ -9,7 +9,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"os/user"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -44,7 +43,7 @@ func main() {
page := 1 page := 1
ratio := 0.0 ratio := 0.0
flag.StringVar(&path, "dir", "unnamed", "Directory to safe pictures. Default is %HOME/pictures/konachan/unnamed") flag.StringVar(&path, "dir", "", "Path to safe pictures. Default is '~/pictures/$site/$tags'")
flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false") flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false")
flag.StringVar(&tags, "tags", "", "Tags used to filter search query.") flag.StringVar(&tags, "tags", "", "Tags used to filter search query.")
flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have") flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have")
@@ -52,19 +51,27 @@ func main() {
flag.Parse() flag.Parse()
// set home directory and create it to save pictures in // get the UserHomeDir
homepath, err := user.Current() homedir, err := os.UserHomeDir()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
filepath := strings.Join([]string{homepath.HomeDir, "pictures", site, strings.TrimSuffix(path, "\n")}, "/")
os.MkdirAll(filepath, 0700)
// edit tags array to met API requirement // edit tags array to met API requirement
tags = strings.Replace(tags, " ", "_", -1)
tags = strings.Replace(tags, ",", "+", -1) tags = strings.Replace(tags, ",", "+", -1)
tags = strings.Replace(tags, "=", ":", -1) tags = strings.Replace(tags, "=", ":", -1)
tags = strings.TrimSuffix(tags, "\n") tags = strings.TrimSuffix(tags, "\n")
// set the path based on variable or tags
default_path := strings.Join([]string{homedir, "pictures", site, tags}, "/")
filepath := default_path
if path != "" {
filepath = path
}
os.MkdirAll(filepath, 0644)
// calculate aspect ratio // calculate aspect ratio
if isFlagPassed("aspect") { if isFlagPassed("aspect") {
aspectSlice := strings.Split(aspect, ":") aspectSlice := strings.Split(aspect, ":")
@@ -79,9 +86,9 @@ func main() {
fmt.Println("Page: ", page) fmt.Println("Page: ", page)
website := fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s", page, tags) website := fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s", page, tags)
if safemode { if safemode {
website = fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s+rating:safe", page, tags) website = fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s+rating:safe", page, tags)
} }
if site == "danbooru" { if site == "danbooru" {
@@ -91,7 +98,16 @@ func main() {
} }
} }
picList := openConnection(website) response := requestAPI(website)
fmt.Println(response.StatusCode)
if response.StatusCode == 421 {
time.Sleep(1 * time.Second)
}
if response.StatusCode != 200 {
defer response.Body.Close()
continue
}
picList := getJSON(response)
pictures, count := parseMaps(picList, ratio) pictures, count := parseMaps(picList, ratio)
picHits = count picHits = count
@@ -117,15 +133,19 @@ func isFlagPassed(name string) bool {
} }
// function to create the connection to konachan and get the API response // function to create the connection to konachan and get the API response
func openConnection(url string) []picture { func requestAPI(url string) *http.Response {
var f []picture
result, err := http.Get(url) result, err := http.Get(url)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
defer result.Body.Close()
return result
}
func getJSON(result *http.Response) []picture {
var f []picture
defer result.Body.Close()
data, err := io.ReadAll(result.Body) data, err := io.ReadAll(result.Body)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)