diff --git a/project.clj b/project.clj index 85e3d8b..9541550 100644 --- a/project.clj +++ b/project.clj @@ -4,6 +4,7 @@ :min-lein-version "2.0.0" :main aretherecookies.app :dependencies [[org.clojure/clojure "1.8.0"] + [org.clojure/core.async "1.2.603"] [environ "1.1.0"] [compojure "1.5.1"] [ring/ring-defaults "0.2.1"] diff --git a/sample.html b/sample.html new file mode 100644 index 0000000..f153b21 --- /dev/null +++ b/sample.html @@ -0,0 +1,13625 @@ + + + + + + + Search Results | Shop H-E-B + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+ Skip to Search Results + +
+
+ +
+Limited availability this week: ordering is now limited to 7 days in advance. You may see limited time slots this week, but more will be available soon.
+
+Limited availability this week: ordering is now limited to 7 days in advance. You may see limited time slots this week, but more will be available soon.
+ +
+
+ + + + + + + + + + + +
+
+ + + + +
+ +
+ + + + + + +
+ + +
+
+
+
+ Filter by
+
+
+ Category
+ +
+
+ +
+
+ Popular filters
+ + + +
+ + +
+
+ Brand
+ + + + + Show all + +
+ + +
+
+ Ways to shop
+ + + + + Show all + +
+ + +
+ +
+ Price Range +
+
+
+ + +
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+

+ “apple” +

+ +
+
+ + + + +
+   +
+ +
+ + +
+ + + 486 results + + + + +
+
+ + +
+ +
+ + + +
+ +
+ + + + +
+ + + + +
+ +
+
+
+
+
+ Recipes +
+
+ + +
+ + + +
+ Load More Items +
Showing 32 of 285
+
+
+ +
+ +
+
+
+ Articles +
+
+ + + + +
+ + + +
+ Load More Items +
Showing 32 of 96
+
+
+ +
+
+
+
+ + + + + + + +
+
+
+
+ + + + + + + + + + + + + + + + diff --git a/scripts/ddl.sql b/scripts/ddl.sql index 4e0718a..d34777c 100644 --- a/scripts/ddl.sql +++ b/scripts/ddl.sql @@ -1,5 +1,5 @@ -CREATE EXTENSION IF NOT EXISTS "postgis"; -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +-- CREATE EXTENSION IF NOT EXISTS "postgis"; +-- CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch"; DROP TYPE IF EXISTS QUANTITY CASCADE; @@ -23,109 +23,51 @@ CREATE TYPE CATEGORY AS ENUM ( 'Toiletries' ); -DROP TABLE IF EXISTS food_items CASCADE; -CREATE TABLE food_items ( - id uuid PRIMARY KEY, - name VARCHAR(100) NOT NULL, - place_id VARCHAR(50) NOT NULL, - category CATEGORY NOT NULL, - images VARCHAR, - loc geography(POINT,4326) +DROP TABLE IF EXISTS product CASCADE; +CREATE TABLE product ( + id VARCHAR(100), + provider_type VARCHAR(50), + name VARCHAR(200), + photo VARCHAR(250), + PRIMARY KEY (id) ); -DROP TABLE IF EXISTS images CASCADE; -CREATE TABLE images ( - filename VARCHAR(200) PRIMARY KEY, - food_item_id uuid REFERENCES food_items (id) ON DELETE CASCADE, - username VARCHAR(100) NOT NULL, - date timestamp (1) with time zone +DROP TABLE IF EXISTS search_term CASCADE; +CREATE TABLE search_term ( + term VARCHAR(200) PRIMARY KEY, + last_queried timestamp (1) with time zone ); -DROP TABLE IF EXISTS quantities CASCADE; -CREATE TABLE quantities ( - food_item_id uuid REFERENCES food_items (id) ON DELETE CASCADE, +DROP TABLE IF EXISTS search_term_product CASCADE; +CREATE TABLE search_term_product ( + term VARCHAR(200) REFERENCES search_term(term) ON DELETE CASCADE, + product_id VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE +); + +DROP TABLE IF EXISTS search_category CASCADE; +CREATE TABLE search_category ( + product_id VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE, + category CATEGORY +); + +DROP TABLE IF EXISTS product_quantity CASCADE; +CREATE TABLE product_quantity ( + product_id VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE, date timestamp (1) with time zone, quantity QUANTITY NOT NULL, - PRIMARY KEY (food_item_id, date) + PRIMARY KEY (product_id, date) ); DROP TABLE IF EXISTS faves CASCADE; CREATE TABLE faves ( - food_item_id uuid REFERENCES food_items (id) ON DELETE CASCADE, + product_id VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE, + photo_url VARCHAR(300), username VARCHAR(200) NOT NULL, date timestamp (1) with time zone NOT NULL, - PRIMARY KEY (food_item_id, username) + PRIMARY KEY (product_id) ); -CREATE INDEX IF NOT EXISTS food_loc_index ON food_items USING GIST ( loc ); - -DROP VIEW IF EXISTS latest_quantities; -CREATE VIEW latest_quantities AS SELECT food_item_id, quantity, date from quantities q1 where date = ( - SELECT max(date) FROM quantities q2 WHERE q1.food_item_id=q2.food_item_id -); - ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Big John Cookies', --- 'ChIJf5QJFBK1RIYRjjfxZz9Z0O0', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.7532464 30.270667599999996)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Jelly Filled Donuts', --- 'ChIJ72if-Qe1RIYRCzMucGEEdBA', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Spelt Brownies', --- 'ChIJG44vBQi1RIYRvWGHdYUolZY', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.7419085 30.265019100000004)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Oatmeal Raisin Cookies', --- 'ChIJf5QJFBK1RIYRjjfxZz9Z0O0', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.7532464 30.270667599999996)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Donuts with Sprinkles', --- 'ChIJ72if-Qe1RIYRCzMucGEEdBA', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Powdered Donuts', --- 'ChIJ72if-Qe1RIYRCzMucGEEdBA', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Snickerdoodles', --- 'ChIJr3szW6a1RIYRkM7LRpnBIO0', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.73798459999999 30.266898599999998)') --- ); ---INSERT INTO food_items (id, name, place_id, category, loc) --- VALUES ( --- uuid_generate_v4(), --- 'Pizza', --- 'ChIJr3szW6a1RIYRkM7LRpnBIO0', --- 'desserts', --- ST_GeogFromText('SRID=4326;POINT(-97.73798459999999 30.266898599999998)') --- ); ---INSERT INTO quantities SELECT id, current_timestamp, 'many' FROM food_items; \ No newline at end of file +CREATE VIEW latest_quantity AS +SELECT DISTINCT ON (product_id) product_id, quantity, date +FROM product_quantity +ORDER BY product_id, date DESC; diff --git a/src/aretherecookies/app.clj b/src/aretherecookies/app.clj index 32e9243..863f9b1 100644 --- a/src/aretherecookies/app.clj +++ b/src/aretherecookies/app.clj @@ -1,14 +1,13 @@ (ns aretherecookies.app (:gen-class) - (:require [aretherecookies.handler :refer [quantity-handler - add-food-item-handler - add-image-handler - get-images-handler + (:require [aretherecookies.handler :refer [quantity-add-handler + quantity-get-handler faves-get-handler faves-put-handler faves-delete-handler - heb-search-handler]] + product-search-handler]] [aretherecookies.auth :refer [auth0-auth-backend]] + [aretherecookies.search.search :refer init-search] [environ.core :refer [env]] [compojure.handler :refer [api]] [compojure.core :refer [defroutes GET POST PUT DELETE]] @@ -22,14 +21,12 @@ (defroutes app-routes (GET "/" [] (str {:csrf-token *anti-forgery-token*})) (POST "/test" [] "ok") - (POST "/quantity" [] quantity-handler) - (POST "/addfooditem" [] add-food-item-handler) - (POST "/images/:foodItemId" [] add-image-handler) - (GET "/images/:foodItemId" [] get-images-handler) + (POST "/quantity" [] quantity-add-handler) + (GET "/quantity" [] quantity-get-handler) (POST "/faves" [] faves-get-handler) (PUT "/fave" [] faves-put-handler) (DELETE "/fave" [] faves-delete-handler) - (GET "/products/:search" [] heb-search-handler)) + (GET "/products/:search" [] product-search-handler)) (def app-config (assoc-in api-defaults [:security :anti-forgery] false)) @@ -38,6 +35,7 @@ (defn -main [& [port]] (let [port (Integer. (or port (env :port) 3000))] + (init-search) (-> (api #'app) (wrap-authentication auth0-auth-backend) diff --git a/src/aretherecookies/auth.clj b/src/aretherecookies/auth.clj index 183df64..7e9a0bd 100644 --- a/src/aretherecookies/auth.clj +++ b/src/aretherecookies/auth.clj @@ -5,8 +5,10 @@ ; TODO implement this as LRU cache to eliminate memory leaking (def tokens (atom {})) + (def auth0-userinfo-url "https://aretherecookies.auth0.com/userinfo") + (defn verify-token "perform rest requesting using token and return boolean if successful" [token url] @@ -17,16 +19,19 @@ (= 200 status)) (catch Exception e (print (str "Caught: " e))))) + (defn cache-token "swap the new token into our cache map atom and return the token" [token] (swap! tokens assoc (keyword token) true)) + (defn verify-and-cache-token "if a REST request is successful against url using token, cache the token and return it" [token url] (if (verify-token token url) (cache-token token) nil)) + (defn url-backend "return a buddy auth backend that validates tokens agianst given url" [url] diff --git a/src/aretherecookies/db.clj b/src/aretherecookies/db.clj index 126178c..f5e849a 100644 --- a/src/aretherecookies/db.clj +++ b/src/aretherecookies/db.clj @@ -2,19 +2,16 @@ (:require [clojure.java.jdbc :as jdbc] [environ.core :refer [env]] [clojure.string :as str] - [hugsql.core :as hugsql] - [clojure.set :refer [rename-keys]]) + [hugsql.core :as hugsql]) (:import com.mchange.v2.c3p0.ComboPooledDataSource)) -; defines a number of functions in this namesapce as defined in queries.sql -; by-distance -; by-last-updated -; by-quantity -; select-food-items -; within-radius -; has-category -; select-images -; update-images +;; -- :name insert-quantity-query +;; -- :name select-quantities-query +;; -- :name select-from-faves-query +;; -- :name insert-into-faves-query +;; -- :name delete-from-faves-query +;; -- :name search-products-by-term +;; -- :name select-search-term (hugsql/def-db-fns "aretherecookies/queries.sql") (def db-spec {:uri (env :database-jdbc-uri) @@ -38,77 +35,63 @@ (defonce pooled-db (delay (pool db-spec))) -(defn get-orderby [{orderby :orderby} & args] - (apply - (cond - (= orderby "distance") by-distance - (= orderby "lastupdated") by-last-updated - (= orderby "quantity") by-quantity - :else by-distance) - args)) (defn wrap-in-quotes [token] (str/replace token #"(^|$)" "'")) -(defn wildcard - [term] - (str "%" (str/join (interpose "%" term)) "%")) -(defn get-where [{:keys [:lat :lng :filter]}] - (food-items-where-clause {:lat lat - :lng lng - :dist (:radius filter) - :categories (map wrap-in-quotes (:categories filter)) - :search (wildcard (:search filter))})) +(defn insert-quantity [{:keys [:productId :quantity]}] + (insert-quantity-query @pooled-db {:product_id productId :quantity quantity})) -(defn query-food-items [{lat :lat lng :lng filter :filter}] - (select-food-items - @pooled-db - {:lat lat - :lng lng - :where (get-where {:lat lat :lng lng :filter filter}) - :order (get-orderby filter)})) -(defn insert-quantity [{:keys [:foodItemId :quantity]}] - (insert-quantity-query @pooled-db {:food_item_id foodItemId :quantity quantity})) +(defn select-quantities + [productIds] + (select-quantities-query @pooled-db {:product_ids productIds})) -(defn select-latest-quantity [{:keys [:foodItemId]}] - (select-latest-quantity-query @pooled-db {:food_item_id (wrap-in-quotes foodItemId)})) - -(defn create-food-item [{:keys [:name :placeId :category :quantity :latitude :longitude]}] - (let [food-item (first (insert-food-item - @pooled-db - {:name name - :placeId placeId - :category category - :longitude longitude - :latitude latitude})) - quantity (first (insert-quantity {:foodItemId (:id food-item) :quantity quantity}))] - (merge - food-item - (rename-keys (select-keys quantity [:date :quantity]) {:date :lastupdated})))) - -(defn get-images - "query database for a list of images for a food item id" - [foodItemId] - (select-images @pooled-db {:foodItemId foodItemId})) - -(defn add-image - "update the list of images for given food item id" - [image] - (insert-image @pooled-db image)) (defn get-faves "select all faves associated with a username" [username] - (select-from-faves @pooled-db {:username username})) + (select-from-faves-query @pooled-db {:username username})) -(defn add-faves + +(defn add-fave "add a food item id to faves for this username" - [username foodItemIds] - (map (fn [foodItemId] (insert-into-faves @pooled-db {:username username :food_item_id foodItemId})) foodItemIds)) + [username productId photoUrl] + (insert-into-faves-query @pooled-db {:username username :product_id productId :photo_url photoUrl})) + (defn delete-faves "remove a list of food item ids from faves for this username" - [username foodItemIds] - (delete-from-faves @pooled-db {:username username :food_item_ids foodItemIds})) + [username productIds] + (delete-from-faves-query @pooled-db {:username username :product_ids productIds})) + + +(defn search-products + "double metaphone search for previous terms mathing search" + [search] + (search-products-by-term @pooled-db {:search search})) + + +(defn get-search-term + "" + [term] + (select-search-term @pooled-db {:term term})) + + +(defn update-search-term + "" + [term] + (insert-into-search-term @pooled-db {:term term})) + + +(defn add-products + "" + [term products] + (println (str "add-products " term)) + (let [product-vecs (map #(vals (select-keys % [:id :placeType :name :photo])) products) + term-pairs (map #(vec [term (:id %)]) products)] + (println "product-vecs" product-vecs) + (println "term-pairs" term-pairs) + (insert-into-product @pooled-db {:products product-vecs}) + (insert-into-search-term-product @pooled-db {:pairs term-pairs}))) diff --git a/src/aretherecookies/handler.clj b/src/aretherecookies/handler.clj index 545d9e2..ab4717d 100644 --- a/src/aretherecookies/handler.clj +++ b/src/aretherecookies/handler.clj @@ -1,24 +1,14 @@ (ns aretherecookies.handler - (:require [aretherecookies.db :refer [query-food-items - insert-quantity - create-food-item - get-images - add-image + (:require [aretherecookies.db :refer [insert-quantity + select-quantities get-faves - add-faves - delete-faves]] - [aretherecookies.parsers :refer [parse-special-types - parse-location]] + add-fave + delete-faves + search-products]] [aretherecookies.helpers :refer [safe-json]] [aretherecookies.search.heb :refer [memoized-search-heb]] - [clojure.data.json :as json] - [aretherecookies.aws :refer [put-s3 - build-s3-url]] - [clojure.string :as str] [buddy.auth :refer [authenticated? - throw-unauthorized]] - [net.cgrand.enlive-html :as html]) - (:import (java.util UUID))) + throw-unauthorized]])) (defn bad-request @@ -27,140 +17,66 @@ {:status 400 :body (safe-json body)}) -(defn update-key-to-s3 - "return a map with key updated to point to s3" - ([key] - (fn [map] - (update map key build-s3-url))) - ([key map] - (update map key build-s3-url))) - - -(defn food-items-handler - "return a list of food items for a given filter and location" +(defn quantity-add-handler + "insert a quantity for a product" [req] - (println "/fooditems ---->" (:body req)) - (let [{body :body} req] - (safe-json - (hash-map - :filter (:filter body) - :fooditems (->> - (query-food-items body) - (map parse-location) - (map (update-key-to-s3 :thumbimage))))))) + (let [{{productId :productId quantity :quantity} :body} req] + (println "/quantity POST ---->" productId quantity) + (if-not (authenticated? req) + (throw-unauthorized) + (safe-json (insert-quantity {:productId productId :quantity quantity}))))) -(defn quantity-handler - "get the latest quantity for a food item" +(defn quantity-get-handler + "get the quantity for a product" [req] - (let [{{foodItemId :foodItemId quantity :quantity} :body} req] - ;(if-not (authenticated? req) (throw-unauthorized)) - (println "/quantity ---->" foodItemId quantity) - (json/write-str - (insert-quantity {:foodItemId foodItemId :quantity quantity}) - :value-fn parse-special-types))) - -(defn key-found? - [obj key] - (not (str/blank? (str (get obj key))))) - -(defn get-missing-keys - "returns a list of required keys which were not found on the given food item" - [foodItem] - (remove - #(key-found? foodItem %) - [:name :placeId :latitude :longitude :category :quantity])) - - -(defn add-food-item-handler - "validate food item fields from request and insert into database returning newly created item" - [req] - (println "/addfooditem ---->" (:body req)) - ;(if-not (authenticated? req) (throw-unauthorized)) - (let [food-item (:body req) missing-keys (get-missing-keys food-item)] - (if (> (count missing-keys) 0) - (bad-request {"missingkeys" missing-keys}) - (as-> - (create-food-item food-item) % - (parse-location %) - (safe-json %))))) - - -(defn get-images-handler - "return all images for a given foodItemId" - [req] - (let [foodItemId (get-in req [:params :foodItemId])] - (->> - (get-images foodItemId) - (map #(update % :url build-s3-url)) - safe-json))) - - -(defn add-image-handler - "given foodItemId from route and photo from multipart form post uploads image into s3 and adds URL into food item record" - [req] - ; (if-not (authenticated? req) (throw-unauthorized)) - (let [username (get-in req [:params :username] "Bart Akeley") - foodItemId (get-in req [:params :foodItemId]) - image (get-in req [:params :photo :tempfile]) - img-name (str foodItemId "/" (UUID/randomUUID) ".jpg")] - (println "/addimage ---->" img-name) - (put-s3 img-name image) - (->> - (add-image {:food_item_id foodItemId - :filename img-name - :username username}) - first - (update-key-to-s3 :url) - safe-json))) + (let [{{productIds :productIds} :body} req] + (println "/quantity GET ---->" productIds) + (if-not (authenticated? req) + (throw-unauthorized) + (safe-json (select-quantities {:productIds productIds}))))) (defn faves-get-handler "return all the faves for a given username" [req] - (if-not (authenticated? req) (throw-unauthorized)) - (let [email (get-in req [:body :email])] - (println "/faves/ ----> POST " email) - (->> (get-faves email) safe-json))) + (if-not (authenticated? req) + (throw-unauthorized) + (let [email (get-in req [:body :email])] + (println "/faves/ ----> POST " email) + (->> (get-faves email) safe-json)))) (defn faves-put-handler "add a food item to faves for a user" [req] - (if-not (authenticated? req) (throw-unauthorized)) - (let [email (get-in req [:body :email]) - foodItemIds (get-in req [:body :foodItemIds])] - (println "/fave/" email "----> PUT" foodItemIds) - (->> (add-faves email foodItemIds) safe-json))) + (if-not (authenticated? req) + (throw-unauthorized) + (let [email (get-in req [:body :email]) + productId (get-in req [:body :productId]) + photoUrl (get-in req [:body :photoUrl])] + (println "/fave/" email "----> PUT" productId photoUrl) + (->> (add-fave email productId photoUrl) safe-json)))) (defn faves-delete-handler "add a food item to faves for a user" [req] - (if-not (authenticated? req) (throw-unauthorized)) - (let [email (get-in req [:body :email]) - foodItemIds (get-in req [:body :foodItemIds])] - (println "/faves/" email "----> DELETE" foodItemIds) - (->> (delete-faves email foodItemIds) safe-json))) + (if-not (authenticated? req) + (throw-unauthorized) + (let [email (get-in req [:body :email]) + productIds (get-in req [:body :productIds])] + (println "/faves/" email "----> DELETE" productIds) + (->> (delete-faves email productIds) safe-json)))) -(defn heb-product-to-food-item - [html-node] - (let [image-url (:data-src (:attrs (first (html/select html-node [:img])))) - content-data (json/read-str (first (:content (first (html/select html-node [:script])))))] - {:id (str "HEB:" (get content-data "id")) - :name (get content-data "name") - :thumbimage image-url - :placeType "HEB" - :category (get content-data "category") - :variant (get content-data "variant")})) - - -(defn heb-search-handler - "passes a search text to heb and parses html response" +(defn product-search-handler + "query cached search results and schedule a cache refresh if needed" [req] (let [search (get-in req [:params :search])] - (println (str "/heb/" search)) + (println (str "/search/" search)) (if-not (empty? search) - (safe-json {:products (memoized-search-heb search)}) + (let [products (search-products search)] + (queue-search search) + (safe-json {:products products}) (safe-json {:products []})))) diff --git a/src/aretherecookies/queries.sql b/src/aretherecookies/queries.sql index 34c662a..6e6cb84 100644 --- a/src/aretherecookies/queries.sql +++ b/src/aretherecookies/queries.sql @@ -1,109 +1,53 @@ --- :name select-food-items :? :raw -SELECT - f.id AS id, - f.name AS name, - f.place_id AS placeid, - f.category AS category, - ST_AsGeoJSON(f.loc) AS location, - ST_Distance(f.loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography) / 1609 AS distance, - q.quantity AS quantity, - q.date AS lastupdated, - (select filename from images i where f.id = i.food_item_id order by date asc limit 1) as thumbimage -FROM food_items f -LEFT OUTER JOIN latest_quantities q -ON f.id = q.food_item_id -:snip:where -:snip:order - --- :snip has-category -WHERE -ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609) -AND -category IN (:i*:categories) - --- :snip within-radius -WHERE ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609) - --- :snip food-items-where-clause -WHERE -ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609) ---~ (when (seq (:categories params)) "AND category IN (:i*:categories)") ---~ (when (not-empty (:search params)) "AND UPPER(name) like UPPER(:v:search)") - --- :snip by-distance -ORDER BY distance ASC - --- :snip by-last-updated -ORDER BY lastUpdated DESC - --- :snip by-quantity -ORDER BY quantity DESC - -- :name insert-quantity-query -INSERT INTO quantities (food_item_id, quantity, date) -VALUES (:v:food_item_id::uuid, :v:quantity::quantity, now()) +INSERT INTO quantities (product_id, quantity, date) +VALUES (:v:product_id, :v:quantity::quantity, now()) RETURNING * --- :name select-latest-quantity-query -SELECT food_item_id, quantity, date AS updated -FROM quantities -WHERE food_item_id=:i:food_item_id -ORDER BY date DESC LIMIT 1 +-- :name select-quantities-query +SELECT product_id, quantity, date +FROM latest_quantities +WHERE product_id IN (:v*:product_ids) --- :name insert-food-item -INSERT INTO food_items (id, name, place_id, category, loc) - VALUES ( - uuid_generate_v4(), - :v:name, - :v:placeId, - :v:category::category, - ST_SetSRID(ST_Point(:longitude, :latitude), 4326)::geography - ) -RETURNING - id AS id, - name AS name, - place_id AS placeid, - category AS category, - ST_AsGeoJSON(loc) AS location, - ST_Distance(loc, ST_SetSRID(ST_Point(:longitude, :latitude), 4326)::geography) / 1609 AS distance - --- :name select-images -SELECT - filename as url, - date, - username, - food_item_id -FROM images -WHERE food_item_id=:v:foodItemId::uuid - --- :name insert-image -INSERT INTO images (filename, username, date, food_item_id) - VALUES ( - :v:filename, - :v:username, - current_timestamp, - :v:food_item_id::uuid - ) -RETURNING - filename as url, - date, - username, - food_item_id - - --- :name select-from-faves -SELECT food_item_id, date +-- :name select-from-faves-query +SELECT product_id, photo_url, date FROM faves WHERE username=:v:username ORDER BY date DESC - --- :name insert-into-faves -INSERT INTO faves (food_item_id, username, date) -VALUES (:v:food_item_id::uuid, :v:username, current_timestamp) +-- :name insert-into-faves-query +INSERT INTO faves (product_id, username, photo_url, date) +VALUES (:v:product_id, :v:username, current_timestamp) ON CONFLICT DO NOTHING -RETURNING food_item_id, date +RETURNING product_id, photo_url, date +-- :name delete-from-faves-query +DELETE FROM faves +WHERE username=:v:username +AND product_id IN (:v*:product_ids) +RETURNING product_id, date --- :name delete-from-faves -DELETE FROM faves WHERE username=:v:username AND food_item_id IN (:v*:food_item_ids::uuid) RETURNING food_item_id, date +-- :name search-products-by-term +SELECT product.* FROM product +INNER JOIN search_term_product on search_term_product=product.id +INNER JOIN search_term ON search_term_product.term=search_term.term +WHERE search_term.term=:v:search +OR dmetaphone(search_term.term)=dmetaphone(:v:search) +OR dmetaphone_alt(search_term.term)=dmetaphone(:v:search) + +-- :name select-search-term +SELECT * FROM search_term WHERE term=:v:term + +-- :name insert-into-product +INSERT INTO product (id, provider_type, name, photo) VALUES :t*:products +ON CONFLICT DO NOTHING +RETURNING id; + +-- :name insert-into-search-term +INSERT INTO search_term (term, last_queried) VALUES (:v:term, now()) +ON CONFLICT (term) DO UPDATE SET last_queried=now() +RETURNING last_queried; + +-- :name insert-into-search-term-product +INSERT INTO search_term_product (term, product_id) VALUES :t*:pairs +ON CONFLICT DO NOTHING +RETURNING term, product_id; diff --git a/src/aretherecookies/scrapers.clj b/src/aretherecookies/scrapers.clj new file mode 100644 index 0000000..e69de29 diff --git a/src/aretherecookies/search/heb.clj b/src/aretherecookies/search/heb.clj index 6955304..395ade0 100644 --- a/src/aretherecookies/search/heb.clj +++ b/src/aretherecookies/search/heb.clj @@ -10,7 +10,7 @@ content-data (json/read-str (first (:content (first (html/select html-node [:script])))))] {:id (str "HEB:" (get content-data "id")) :name (get content-data "name") - :thumbimage image-url + :photo image-url :placeType "HEB" :category (get content-data "category") :variant (get content-data "variant")})) diff --git a/src/aretherecookies/search/search.clj b/src/aretherecookies/search/search.clj new file mode 100644 index 0000000..bb23034 --- /dev/null +++ b/src/aretherecookies/search/search.clj @@ -0,0 +1,31 @@ +(ns aretherecookies.search.search + (:require [clojure.core.async :as async :refer [chan, sliding-buffer, go, >!, ! searches-chan term) nil))) + + +(defn search-all-sources + "" + [term] + (let [heb-result (go (search-heb term))] + (update-search-term term) + (async/map #(add-products term %) [heb-result]))) + + +(defn init-search + "" + [] + (go (while true + (try (search-all-sources (