Mel/imagecrawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change default save location and add go mod

Browse Source
This commit is contained in:
m3philis
2025-06-01 16:32:48 +02:00
parent 693290bc03
commit 00d25bbe3a
3 changed files with 36 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
imagecrawler.go
View File

@@ -9,7 +9,6 @@ import (
"net/http"
"net/url"
"os"
"os/user"
"strconv"
"strings"
"sync"
@@ -44,7 +43,7 @@ func main() {
page := 1
ratio := 0.0
flag.StringVar(&path, "dir", "unnamed", "Directory to safe pictures. Default is %HOME/pictures/konachan/unnamed")
flag.StringVar(&path, "dir", "", "Path to safe pictures. Default is '~/pictures/$site/$tags'")
flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false")
flag.StringVar(&tags, "tags", "", "Tags used to filter search query.")
flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have")
@@ -52,19 +51,27 @@ func main() {
flag.Parse()
// set home directory and create it to save pictures in
homepath, err := user.Current()
// get the UserHomeDir
homedir, err := os.UserHomeDir()
if err != nil {
log.Fatal(err)
}
filepath := strings.Join([]string{homepath.HomeDir, "pictures", site, strings.TrimSuffix(path, "\n")}, "/")
os.MkdirAll(filepath, 0700)
// edit tags array to met API requirement
tags = strings.Replace(tags, " ", "_", -1)
tags = strings.Replace(tags, ",", "+", -1)
tags = strings.Replace(tags, "=", ":", -1)
tags = strings.TrimSuffix(tags, "\n")
// set the path based on variable or tags
default_path := strings.Join([]string{homedir, "pictures", site, tags}, "/")
filepath := default_path
if path != "" {
filepath = path
}
os.MkdirAll(filepath, 0644)
// calculate aspect ratio
if isFlagPassed("aspect") {
aspectSlice := strings.Split(aspect, ":")
@@ -79,9 +86,9 @@ func main() {
fmt.Println("Page: ", page)
website := fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s", page, tags)
website := fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s", page, tags)
if safemode {
website = fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s+rating:safe", page, tags)
website = fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s+rating:safe", page, tags)
}
if site == "danbooru" {
@@ -91,7 +98,16 @@ func main() {
}
}
picList := openConnection(website)
response := requestAPI(website)
fmt.Println(response.StatusCode)
if response.StatusCode == 421 {
time.Sleep(1 * time.Second)
}
if response.StatusCode != 200 {
defer response.Body.Close()
continue
}
picList := getJSON(response)
pictures, count := parseMaps(picList, ratio)
picHits = count
@@ -117,15 +133,19 @@ func isFlagPassed(name string) bool {
}
// function to create the connection to konachan and get the API response
func openConnection(url string) []picture {
var f []picture
func requestAPI(url string) *http.Response {
result, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer result.Body.Close()
return result
}
func getJSON(result *http.Response) []picture {
var f []picture
defer result.Body.Close()
data, err := io.ReadAll(result.Body)
if err != nil {
log.Fatal(err)