From 1508694b9543df8e020a3baadc4bd66110b757b8 Mon Sep 17 00:00:00 2001 From: ParkSuMin Date: Tue, 7 Jan 2025 15:00:43 +0000 Subject: [PATCH 1/5] Database module --- config/INSTRUCTIONS.md | 12 ++++++ database/database.go | 85 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 6 +++ go.sum | 8 ++++ 4 files changed, 111 insertions(+) create mode 100644 config/INSTRUCTIONS.md create mode 100644 database/database.go diff --git a/config/INSTRUCTIONS.md b/config/INSTRUCTIONS.md new file mode 100644 index 00000000..08359474 --- /dev/null +++ b/config/INSTRUCTIONS.md @@ -0,0 +1,12 @@ +# Example for dbconfig.yaml + +```yaml +user: "user" +password: "password" +dbname: "test" +sslmode: "disable" +hostname: "localhost" +``` +We use Postgres v. 16 + +You need to create own yaml file \ No newline at end of file diff --git a/database/database.go b/database/database.go new file mode 100644 index 00000000..425c5ea3 --- /dev/null +++ b/database/database.go @@ -0,0 +1,85 @@ +package database + +import ( + "fmt" + "log" + "os" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + yaml "gopkg.in/yaml.v2" +) + +type Conf struct { + User string `yaml:"user"` + DBname string `yaml:"dbname"` + SSLMode string `yaml:"sslmode"` + Password string `yaml:"password"` + Host string `yaml:"hostname"` +} + +func LoadConfig(path string) (*Conf, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var config Conf + err = yaml.Unmarshal(data, &config) + if err != nil { + return nil, err + } + return &config, nil +} + +func CheckError(err error) bool { + if err != nil { + log.Fatalln(err) + return false + } + return true +} + +func Test_connection() { + config, err := LoadConfig("config/config.yaml") + if err != nil { + log.Fatalln("Failed to load config:", err) + } + dsn := fmt.Sprintf("user=%s dbname=%s sslmode=%s password=%s host=%s", + config.User, config.DBname, config.SSLMode, config.Password, config.Host) + + db, err := sqlx.Connect("postgres", dsn) + if err != nil { + log.Fatalln(err) + } + + defer db.Close() + + // Test the connection to the database + if err := db.Ping(); err != nil { + log.Fatal(err) + } else { + log.Println("Successfully Connected") + } +} + +func Insert_Data(query string) bool { + config, err := LoadConfig("config/config.yaml") + if err != nil { + log.Fatalln("Failed to load config:", err) + return false + } + + dsn := fmt.Sprintf("user=%s dbname=%s sslmode=%s password=%s host=%s", + config.User, config.DBname, config.SSLMode, config.Password, config.Host) + + db, err := sqlx.Open("postgres", dsn) + if !CheckError(err) { + return false + } + + defer db.Close() + + _, e := db.Exec(query) + + return CheckError(e) +} diff --git a/go.mod b/go.mod index 9a36617e..5288bc06 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ module market go 1.18 + +require ( + github.com/jmoiron/sqlx v1.4.0 + github.com/lib/pq v1.10.9 + gopkg.in/yaml.v2 v2.4.0 +) \ No newline at end of file diff --git a/go.sum b/go.sum index 36c43246..951f03ae 100644 --- a/go.sum +++ b/go.sum @@ -6,5 +6,13 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 1698c85bd66572e05e6a05dcd32d791530d128bd Mon Sep 17 00:00:00 2001 From: ParkSuMin Date: Tue, 7 Jan 2025 15:25:59 +0000 Subject: [PATCH 2/5] Base of parser (currency part) --- .gitignore | 2 +- parser/currency.py | 20 ++++++++++++++++++++ parser/requirements.txt | 3 +++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 parser/currency.py create mode 100644 parser/requirements.txt diff --git a/.gitignore b/.gitignore index 2206544d..02932dce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ - +.venv .codegpt \ No newline at end of file diff --git a/parser/currency.py b/parser/currency.py new file mode 100644 index 00000000..01354f96 --- /dev/null +++ b/parser/currency.py @@ -0,0 +1,20 @@ +import requests +import json + +url = "https://www.cbr-xml-daily.ru/daily_json.js" + +class Currency: + def __init__(self, code: str): + self.name = code + self.status = True + response = requests.get(url) + response.raise_for_status() # Проверяем успешность запроса + try: + # Парсим JSON-данные + data = response.json() + self.yen_rate = data["Valute"][code]["Value"] + self.yen_nominal = data["Valute"][code]["Nominal"] + except: + self.status = False + +obj = Currency("JPY") \ No newline at end of file diff --git a/parser/requirements.txt b/parser/requirements.txt new file mode 100644 index 00000000..d25f1dae --- /dev/null +++ b/parser/requirements.txt @@ -0,0 +1,3 @@ +bs4==0.0.2 +requests==2.32.3 + From 79aa5bab48f47e6871c25110de5d776a139eb0da Mon Sep 17 00:00:00 2001 From: ParkSuMin Date: Tue, 7 Jan 2025 15:43:47 +0000 Subject: [PATCH 3/5] Prepare for parser's part --- parser/main.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 parser/main.py diff --git a/parser/main.py b/parser/main.py new file mode 100644 index 00000000..0892ad4f --- /dev/null +++ b/parser/main.py @@ -0,0 +1,17 @@ +from currency import Currency +import requests +from bs4 import BeautifulSoup + +url = "https://irose-shop.net/?category_id=52b1abaf8a56103f0500013e" + +response = requests.get(url) +response.raise_for_status() + +soup = BeautifulSoup(response.text, "html.parser") + +products = soup.find_all("div", class_="c-itemList__item") +print(products) +# bank = Currency("JPY") +# price = 19160 + +# print(f"Price in Russian Rubles: {price * ((bank.yen_rate/bank.yen_nominal))}") \ No newline at end of file From b2fd821b3c3d0ec5b9e8dbf81ebbf1480d0816d2 Mon Sep 17 00:00:00 2001 From: ParkSuMin Date: Tue, 7 Jan 2025 20:40:13 +0000 Subject: [PATCH 4/5] Example of fint-shop's parsing --- .gitignore | 3 ++- parser/currency.py | 13 +++++-------- parser/item.py | 6 ++++++ parser/main.py | 43 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 parser/item.py diff --git a/.gitignore b/.gitignore index 02932dce..b4be74b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .venv -.codegpt \ No newline at end of file +.codegpt +__pycache__ \ No newline at end of file diff --git a/parser/currency.py b/parser/currency.py index 01354f96..3036404a 100644 --- a/parser/currency.py +++ b/parser/currency.py @@ -1,20 +1,17 @@ import requests -import json url = "https://www.cbr-xml-daily.ru/daily_json.js" + class Currency: def __init__(self, code: str): self.name = code self.status = True response = requests.get(url) - response.raise_for_status() # Проверяем успешность запроса + response.raise_for_status() try: - # Парсим JSON-данные data = response.json() - self.yen_rate = data["Valute"][code]["Value"] - self.yen_nominal = data["Valute"][code]["Nominal"] - except: + self.rate = data["Valute"][code]["Value"] + self.nominal = data["Valute"][code]["Nominal"] + except Exception: self.status = False - -obj = Currency("JPY") \ No newline at end of file diff --git a/parser/item.py b/parser/item.py new file mode 100644 index 00000000..e61b3c9a --- /dev/null +++ b/parser/item.py @@ -0,0 +1,6 @@ +class Item: + def __init__(self, name: str, category: str, price: float, photo_url: str): + self.name = name + self.category = category + self.price = price + self.photo_url = photo_url diff --git a/parser/main.py b/parser/main.py index 0892ad4f..a83e9e1c 100644 --- a/parser/main.py +++ b/parser/main.py @@ -1,17 +1,46 @@ -from currency import Currency +import sys + import requests from bs4 import BeautifulSoup +from currency import Currency +from item import Item -url = "https://irose-shop.net/?category_id=52b1abaf8a56103f0500013e" +items = [] + +bank = Currency("JPY") +if not bank.status: + print("Not correct currency code") + sys.exit() + +url = "https://www.fint-shop.com/c/all/bag?top_category" response = requests.get(url) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") -products = soup.find_all("div", class_="c-itemList__item") -print(products) -# bank = Currency("JPY") -# price = 19160 +products = soup.find_all("article", class_="fs-c-productList__list__item") +category_name = soup.find_all("li", class_="fs-c-breadcrumb__listItem")[1].get_text( + strip=True +) +print(category_name) +for product in products: + product_name = product.find("span", class_="fs-c-productName__name").get_text( + strip=True + ) -# print(f"Price in Russian Rubles: {price * ((bank.yen_rate/bank.yen_nominal))}") \ No newline at end of file + price = int( + product.find("div", class_="fs-c-productPrice--listed") + .find("span", class_="fs-c-price__value") + .get_text(strip=True) + .replace(",", "") + ) + product_price = round(price * (bank.rate / bank.nominal), 2) + + product_photo = product.find( + "img", class_="fs-c-productListItem__image__image fs-c-productImage__image" + )["data-layzr"] + items.append(Item(product_name, "bags", product_price, product_photo)) + +for item in items: + print(f"Name = {item.name}, price = {item.price}") From 4e12e6717e992ddd903b4dd9b2aae0433abb2483 Mon Sep 17 00:00:00 2001 From: ParkSuMin Date: Tue, 7 Jan 2025 20:42:20 +0000 Subject: [PATCH 5/5] pre-commit tool --- .pre-commit-config.yaml | 11 +++++++++++ parser/requirements.txt | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..63f1b24d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.0 + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort \ No newline at end of file diff --git a/parser/requirements.txt b/parser/requirements.txt index d25f1dae..b5cb0f06 100644 --- a/parser/requirements.txt +++ b/parser/requirements.txt @@ -1,3 +1,3 @@ bs4==0.0.2 -requests==2.32.3 - +pre_commit==4.0.1 +requests==2.32.3 \ No newline at end of file