change default save location and add go mod
This commit is contained in:
BIN
imagecrawler
Executable file
BIN
imagecrawler
Executable file
Binary file not shown.
@@ -9,7 +9,6 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"os/user"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -44,7 +43,7 @@ func main() {
|
|||||||
page := 1
|
page := 1
|
||||||
ratio := 0.0
|
ratio := 0.0
|
||||||
|
|
||||||
flag.StringVar(&path, "dir", "unnamed", "Directory to safe pictures. Default is %HOME/pictures/konachan/unnamed")
|
flag.StringVar(&path, "dir", "", "Path to safe pictures. Default is '~/pictures/$site/$tags'")
|
||||||
flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false")
|
flag.BoolVar(&safemode, "safe", false, "Safemode to filter NSFW pictures. Default is false")
|
||||||
flag.StringVar(&tags, "tags", "", "Tags used to filter search query.")
|
flag.StringVar(&tags, "tags", "", "Tags used to filter search query.")
|
||||||
flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have")
|
flag.StringVar(&aspect, "aspect", "", "Aspect ratio pics should have")
|
||||||
@@ -52,19 +51,27 @@ func main() {
|
|||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// set home directory and create it to save pictures in
|
// get the UserHomeDir
|
||||||
homepath, err := user.Current()
|
homedir, err := os.UserHomeDir()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
filepath := strings.Join([]string{homepath.HomeDir, "pictures", site, strings.TrimSuffix(path, "\n")}, "/")
|
|
||||||
os.MkdirAll(filepath, 0700)
|
|
||||||
|
|
||||||
// edit tags array to met API requirement
|
// edit tags array to met API requirement
|
||||||
|
tags = strings.Replace(tags, " ", "_", -1)
|
||||||
tags = strings.Replace(tags, ",", "+", -1)
|
tags = strings.Replace(tags, ",", "+", -1)
|
||||||
tags = strings.Replace(tags, "=", ":", -1)
|
tags = strings.Replace(tags, "=", ":", -1)
|
||||||
tags = strings.TrimSuffix(tags, "\n")
|
tags = strings.TrimSuffix(tags, "\n")
|
||||||
|
|
||||||
|
// set the path based on variable or tags
|
||||||
|
default_path := strings.Join([]string{homedir, "pictures", site, tags}, "/")
|
||||||
|
filepath := default_path
|
||||||
|
if path != "" {
|
||||||
|
filepath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
os.MkdirAll(filepath, 0644)
|
||||||
|
|
||||||
// calculate aspect ratio
|
// calculate aspect ratio
|
||||||
if isFlagPassed("aspect") {
|
if isFlagPassed("aspect") {
|
||||||
aspectSlice := strings.Split(aspect, ":")
|
aspectSlice := strings.Split(aspect, ":")
|
||||||
@@ -79,9 +86,9 @@ func main() {
|
|||||||
|
|
||||||
fmt.Println("Page: ", page)
|
fmt.Println("Page: ", page)
|
||||||
|
|
||||||
website := fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s", page, tags)
|
website := fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s", page, tags)
|
||||||
if safemode {
|
if safemode {
|
||||||
website = fmt.Sprintf("https://konachan.com/post.json?page=%d&tags=%s+rating:safe", page, tags)
|
website = fmt.Sprintf("https://konachan.com/post.json?limit=100&page=%d&tags=%s+rating:safe", page, tags)
|
||||||
}
|
}
|
||||||
|
|
||||||
if site == "danbooru" {
|
if site == "danbooru" {
|
||||||
@@ -91,7 +98,16 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
picList := openConnection(website)
|
response := requestAPI(website)
|
||||||
|
fmt.Println(response.StatusCode)
|
||||||
|
if response.StatusCode == 421 {
|
||||||
|
time.Sleep(1 * time.Second)
|
||||||
|
}
|
||||||
|
if response.StatusCode != 200 {
|
||||||
|
defer response.Body.Close()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
picList := getJSON(response)
|
||||||
pictures, count := parseMaps(picList, ratio)
|
pictures, count := parseMaps(picList, ratio)
|
||||||
|
|
||||||
picHits = count
|
picHits = count
|
||||||
@@ -117,15 +133,19 @@ func isFlagPassed(name string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// function to create the connection to konachan and get the API response
|
// function to create the connection to konachan and get the API response
|
||||||
func openConnection(url string) []picture {
|
func requestAPI(url string) *http.Response {
|
||||||
var f []picture
|
|
||||||
|
|
||||||
result, err := http.Get(url)
|
result, err := http.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
defer result.Body.Close()
|
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func getJSON(result *http.Response) []picture {
|
||||||
|
var f []picture
|
||||||
|
|
||||||
|
defer result.Body.Close()
|
||||||
data, err := io.ReadAll(result.Body)
|
data, err := io.ReadAll(result.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
|
|||||||
Reference in New Issue
Block a user