progress on convert()

go.mod

@@ -4,8 +4,11 @@ go 1.23.1
 
 require github.com/ncruces/go-sqlite3 v0.18.3
 
+require golang.org/x/sync v0.8.0 // indirect
+
 require (
 	github.com/ncruces/julianday v1.0.0 // indirect
+	github.com/otiai10/copy v1.14.0
 	github.com/tetratelabs/wazero v1.8.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
 )

go.sum

@@ -2,7 +2,11 @@ github.com/ncruces/go-sqlite3 v0.18.3 h1:tyMa75uh7LcINcfo0WrzOvcTkfz8Hqu0TEPX+KV
 github.com/ncruces/go-sqlite3 v0.18.3/go.mod h1:HAwOtA+cyEX3iN6YmkpQwfT4vMMgCB7rQRFUdOgEFik=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
 github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
 github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"fmt"
 	"log"
 	"net/url"
 	"os"
@@ -9,6 +10,7 @@ import (
 
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
+	// "github.com/otiai10/copy"
 )
 
 const memory = ":memory:"
@@ -88,16 +90,31 @@ func getArtistFromUrl(Url string) string {
 		}
 		return unescapedUrl[51:]
 	}
-	if strings.Contains(Url, "kemono.su") {
+	if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") {
-		return "Kemono"
+		return strings.Split(Url, "/")[5]
-	}
-	if strings.Contains(Url, "coomer.su") {
-		return "Coomer"
 	}
 
 	return ""
 }
 
+func convert(fileLine string) error {
+	log.Println(fileLine)
+	if strings.Contains(fileLine, "%") {
+		// we can look for specific encoded chars and the remove them and continue instead of returning an error
+		return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made")
+	}
+	artist := getArtistFromUrl(fileLine)
+	log.Println("artist:", artist)
+	//path := fmt.Sprintf("Artists/%s", artist)
+	//log.Println("path:", path)
+
+	if strings.ContainsAny(artist, "!\"#$&'()*,;<>?[\\]^`{|}~") {
+		return fmt.Errorf("%s", "artistName contains disallowed characters and a direct conversion cannot be reasonably made")
+	}
+
+	return nil
+}
+
 func parseLinks(filename string) error {
 	file, err := os.Open(filename)
 	if err != nil {
@@ -106,14 +123,27 @@ func parseLinks(filename string) error {
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		line := scanner.Text()
+		var line string
+		line = scanner.Text()
+
 		if strings.HasPrefix(line, "#") {
 			continue
 		}
-		if !strings.Contains(line, "DBPath") {
+		if !strings.Contains(line, "#") {
+			err = convert(line)
+			if err != nil {
+				log.Printf("error: %s\n", err)
+				continue
+			}
 			continue
 		}
+		/*
+		split_url := strings.Split(line, "#")
+		url := strings.TrimSpace(split_url[0])
+		artist := strings.TrimSpace(split_url[1])
 
+		log.Printf("url: %s\tartist: %s\n", url, artist)
+		*/
 	}
 
 	err = file.Close()
@@ -136,8 +166,10 @@ func main() {
 		log.Fatal(err)
 	}
 
-	artist := getArtistFromUrl("https://rule34.xxx/index.php?page=post&s=list&tags=%27eadole")
+	err = parseLinks("gallery-dl/Rule34.links")
-	log.Println("artist from url: ", artist)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	err = db.Close()
 	if err != nil {