wip converting to products from scrapers

project.clj

@@ -4,6 +4,7 @@
   :min-lein-version "2.0.0"
   :main aretherecookies.app
   :dependencies [[org.clojure/clojure "1.8.0"]
+                 [org.clojure/core.async "1.2.603"]
                  [environ "1.1.0"]
                  [compojure "1.5.1"]
                  [ring/ring-defaults "0.2.1"]

scripts/ddl.sql

@@ -1,5 +1,5 @@
-CREATE EXTENSION IF NOT EXISTS "postgis";
+-- CREATE EXTENSION IF NOT EXISTS "postgis";
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+-- CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 CREATE EXTENSION IF NOT EXISTS "fuzzystrmatch";
 
 DROP TYPE IF EXISTS QUANTITY CASCADE;
@@ -23,109 +23,51 @@ CREATE TYPE CATEGORY AS ENUM (
     'Toiletries'
 );
 
-DROP TABLE IF EXISTS food_items CASCADE;
+DROP TABLE IF EXISTS product CASCADE;
-CREATE TABLE food_items (
+CREATE TABLE product (
-	id              uuid PRIMARY KEY,
+    id     VARCHAR(100),
-	name            VARCHAR(100) NOT NULL,
+    provider_type      VARCHAR(50),
-	place_id        VARCHAR(50) NOT NULL,
+    name               VARCHAR(200),
-	category        CATEGORY NOT NULL,
+    photo              VARCHAR(250),
-	images          VARCHAR,
+    PRIMARY KEY (id)
-	loc             geography(POINT,4326)
 );
 
-DROP TABLE IF EXISTS images CASCADE;
+DROP TABLE IF EXISTS search_term CASCADE;
-CREATE TABLE images (
+CREATE TABLE search_term (
-    filename        VARCHAR(200) PRIMARY KEY,
+    term            VARCHAR(200) PRIMARY KEY,
-    food_item_id    uuid REFERENCES food_items (id) ON DELETE CASCADE,
+    last_queried    timestamp (1) with time zone
-    username        VARCHAR(100) NOT NULL,
-    date            timestamp (1) with time zone
 );
 
-DROP TABLE IF EXISTS quantities CASCADE;
+DROP TABLE IF EXISTS search_term_product CASCADE;
-CREATE TABLE quantities (
+CREATE TABLE search_term_product (
-	food_item_id    uuid REFERENCES food_items (id) ON DELETE CASCADE,
+    term        VARCHAR(200) REFERENCES search_term(term) ON DELETE CASCADE,
+    product_id  VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE
+);
+
+DROP TABLE IF EXISTS search_category CASCADE;
+CREATE TABLE search_category (
+    product_id      VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE,
+    category        CATEGORY
+);
+
+DROP TABLE IF EXISTS product_quantity CASCADE;
+CREATE TABLE product_quantity (
+	product_id      VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE,
 	date            timestamp (1) with time zone,
 	quantity        QUANTITY NOT NULL,
-	PRIMARY KEY (food_item_id, date)
+	PRIMARY KEY (product_id, date)
 );
 
 DROP TABLE IF EXISTS faves CASCADE;
 CREATE TABLE faves (
-    food_item_id    uuid REFERENCES food_items (id) ON DELETE CASCADE,
+    product_id      VARCHAR(100) REFERENCES product(id) ON DELETE CASCADE,
+    photo_url       VARCHAR(300),
     username        VARCHAR(200) NOT NULL,
     date            timestamp (1) with time zone NOT NULL,
-    PRIMARY KEY (food_item_id, username)
+    PRIMARY KEY (product_id)
 );
 
-CREATE INDEX IF NOT EXISTS food_loc_index ON food_items USING GIST ( loc );
+CREATE VIEW latest_quantity AS
-
+SELECT DISTINCT ON (product_id) product_id, quantity, date
-DROP VIEW IF EXISTS latest_quantities;
+FROM product_quantity
-CREATE VIEW latest_quantities AS SELECT food_item_id, quantity, date from quantities q1 where date = (
+ORDER BY product_id, date DESC;
-	SELECT max(date) FROM quantities q2 WHERE q1.food_item_id=q2.food_item_id
-);
-
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Big John Cookies',
---		'ChIJf5QJFBK1RIYRjjfxZz9Z0O0',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.7532464 30.270667599999996)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Jelly Filled Donuts',
---		'ChIJ72if-Qe1RIYRCzMucGEEdBA',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Spelt Brownies',
---		'ChIJG44vBQi1RIYRvWGHdYUolZY',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.7419085 30.265019100000004)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Oatmeal Raisin Cookies',
---		'ChIJf5QJFBK1RIYRjjfxZz9Z0O0',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.7532464 30.270667599999996)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Donuts with Sprinkles',
---		'ChIJ72if-Qe1RIYRCzMucGEEdBA',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Powdered Donuts',
---		'ChIJ72if-Qe1RIYRCzMucGEEdBA',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.742308 30.263963300000004)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Snickerdoodles',
---		'ChIJr3szW6a1RIYRkM7LRpnBIO0',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.73798459999999 30.266898599999998)')
---	);
---INSERT INTO food_items (id, name, place_id, category, loc)
---  VALUES (
---		uuid_generate_v4(),
---		'Pizza',
---		'ChIJr3szW6a1RIYRkM7LRpnBIO0',
---		'desserts',
---		ST_GeogFromText('SRID=4326;POINT(-97.73798459999999 30.266898599999998)')
---	);
---INSERT INTO quantities SELECT id, current_timestamp, 'many' FROM food_items;

src/aretherecookies/app.clj

@@ -1,14 +1,13 @@
 (ns aretherecookies.app
   (:gen-class)
-  (:require [aretherecookies.handler :refer [quantity-handler
+  (:require [aretherecookies.handler :refer [quantity-add-handler
-                                             add-food-item-handler
+                                             quantity-get-handler
-                                             add-image-handler
-                                             get-images-handler
                                              faves-get-handler
                                              faves-put-handler
                                              faves-delete-handler
-                                             heb-search-handler]]
+                                             product-search-handler]]
             [aretherecookies.auth :refer [auth0-auth-backend]]
+            [aretherecookies.search.search :refer init-search]
             [environ.core :refer [env]]
             [compojure.handler :refer [api]]
             [compojure.core :refer [defroutes GET POST PUT DELETE]]
@@ -22,14 +21,12 @@
 (defroutes app-routes
   (GET "/" [] (str {:csrf-token *anti-forgery-token*}))
   (POST "/test" [] "ok")
-  (POST "/quantity" [] quantity-handler)
+  (POST "/quantity" [] quantity-add-handler)
-  (POST "/addfooditem" [] add-food-item-handler)
+  (GET "/quantity" [] quantity-get-handler)
-  (POST "/images/:foodItemId" [] add-image-handler)
-  (GET "/images/:foodItemId" [] get-images-handler)
   (POST "/faves" [] faves-get-handler)
   (PUT "/fave" [] faves-put-handler)
   (DELETE "/fave" [] faves-delete-handler)
-  (GET "/products/:search" [] heb-search-handler))
+  (GET "/products/:search" [] product-search-handler))
 
 (def app-config (assoc-in api-defaults [:security :anti-forgery] false))
 
@@ -38,6 +35,7 @@
 (defn -main
   [& [port]]
   (let [port (Integer. (or port (env :port) 3000))]
+    (init-search)
     (->
      (api #'app)
      (wrap-authentication auth0-auth-backend)

src/aretherecookies/auth.clj

@@ -5,8 +5,10 @@
 ; TODO implement this as LRU cache to eliminate memory leaking
 (def tokens (atom {}))
 
+
 (def auth0-userinfo-url "https://aretherecookies.auth0.com/userinfo")
 
+
 (defn verify-token
   "perform rest requesting using token and return boolean if successful"
   [token url]
@@ -17,16 +19,19 @@
       (= 200 status))
     (catch Exception e (print (str "Caught: " e)))))
 
+
 (defn cache-token
   "swap the new token into our cache map atom and return the token"
   [token]
   (swap! tokens assoc (keyword token) true))
 
+
 (defn verify-and-cache-token
   "if a REST request is successful against url using token, cache the token and return it"
   [token url]
   (if (verify-token token url) (cache-token token) nil))
 
+
 (defn url-backend
   "return a buddy auth backend that validates tokens agianst given url"
   [url]

src/aretherecookies/db.clj

@@ -2,19 +2,16 @@
   (:require [clojure.java.jdbc :as jdbc]
             [environ.core :refer [env]]
             [clojure.string :as str]
-            [hugsql.core :as hugsql]
+            [hugsql.core :as hugsql])
-            [clojure.set :refer [rename-keys]])
   (:import com.mchange.v2.c3p0.ComboPooledDataSource))
 
-; defines a number of functions in this namesapce as defined in queries.sql
+;; -- :name insert-quantity-query
-; by-distance
+;; -- :name select-quantities-query
-; by-last-updated
+;; -- :name select-from-faves-query
-; by-quantity
+;; -- :name insert-into-faves-query
-; select-food-items
+;; -- :name delete-from-faves-query
-; within-radius
+;; -- :name search-products-by-term
-; has-category
+;; -- :name select-search-term
-; select-images
-; update-images
 (hugsql/def-db-fns "aretherecookies/queries.sql")
 
 (def db-spec {:uri (env :database-jdbc-uri)
@@ -38,77 +35,63 @@
 
 (defonce pooled-db (delay (pool db-spec)))
 
-(defn get-orderby [{orderby :orderby} & args]
-  (apply
-   (cond
-     (= orderby "distance") by-distance
-     (= orderby "lastupdated") by-last-updated
-     (= orderby "quantity") by-quantity
-     :else by-distance)
-   args))
 
 (defn wrap-in-quotes [token]
   (str/replace token #"(^|$)" "'"))
 
-(defn wildcard
-  [term]
-  (str "%" (str/join (interpose "%" term)) "%"))
 
-(defn get-where [{:keys [:lat :lng :filter]}]
+(defn insert-quantity [{:keys [:productId :quantity]}]
-  (food-items-where-clause {:lat lat
+  (insert-quantity-query @pooled-db {:product_id productId :quantity quantity}))
-                            :lng lng
-                            :dist (:radius filter)
-                            :categories (map wrap-in-quotes (:categories filter))
-                            :search (wildcard (:search filter))}))
 
-(defn query-food-items [{lat :lat lng :lng filter :filter}]
-  (select-food-items
-   @pooled-db
-   {:lat lat
-    :lng lng
-    :where (get-where {:lat lat :lng lng :filter filter})
-    :order (get-orderby filter)}))
 
-(defn insert-quantity [{:keys [:foodItemId :quantity]}]
+(defn select-quantities
-  (insert-quantity-query @pooled-db {:food_item_id foodItemId :quantity quantity}))
+  [productIds]
+  (select-quantities-query @pooled-db {:product_ids productIds}))
 
-(defn select-latest-quantity [{:keys [:foodItemId]}]
-  (select-latest-quantity-query @pooled-db {:food_item_id (wrap-in-quotes foodItemId)}))
-
-(defn create-food-item [{:keys [:name :placeId :category :quantity :latitude :longitude]}]
-  (let [food-item (first (insert-food-item
-                          @pooled-db
-                          {:name name
-                           :placeId placeId
-                           :category category
-                           :longitude longitude
-                           :latitude latitude}))
-        quantity (first (insert-quantity {:foodItemId (:id food-item) :quantity quantity}))]
-    (merge
-     food-item
-     (rename-keys (select-keys quantity [:date :quantity]) {:date :lastupdated}))))
-
-(defn get-images
-  "query database for a list of images for a food item id"
-  [foodItemId]
-  (select-images @pooled-db {:foodItemId foodItemId}))
-
-(defn add-image
-  "update the list of images for given food item id"
-  [image]
-  (insert-image @pooled-db image))
 
 (defn get-faves
   "select all faves associated with a username"
   [username]
-  (select-from-faves @pooled-db {:username username}))
+  (select-from-faves-query @pooled-db {:username username}))
 
-(defn add-faves
+
+(defn add-fave
   "add a food item id to faves for this username"
-  [username foodItemIds]
+  [username productId photoUrl]
-  (map (fn [foodItemId] (insert-into-faves @pooled-db {:username username :food_item_id foodItemId})) foodItemIds))
+  (insert-into-faves-query @pooled-db {:username username :product_id productId :photo_url photoUrl}))
+
 
 (defn delete-faves
   "remove a list of food item ids from faves for this username"
-  [username foodItemIds]
+  [username productIds]
-  (delete-from-faves @pooled-db {:username username :food_item_ids foodItemIds}))
+  (delete-from-faves-query @pooled-db {:username username :product_ids productIds}))
+
+
+(defn search-products
+  "double metaphone search for previous terms mathing search"
+  [search]
+  (search-products-by-term @pooled-db {:search search}))
+
+
+(defn get-search-term
+  ""
+  [term]
+  (select-search-term @pooled-db {:term term}))
+
+
+(defn update-search-term
+  ""
+  [term]
+  (insert-into-search-term @pooled-db {:term term}))
+
+
+(defn add-products
+  ""
+  [term products]
+  (println (str "add-products " term))
+  (let [product-vecs (map #(vals (select-keys % [:id :placeType :name :photo])) products)
+        term-pairs (map #(vec [term (:id %)]) products)]
+    (println "product-vecs" product-vecs)
+    (println "term-pairs" term-pairs)
+    (insert-into-product @pooled-db {:products product-vecs})
+    (insert-into-search-term-product @pooled-db {:pairs term-pairs})))

src/aretherecookies/handler.clj

@@ -1,24 +1,14 @@
 (ns aretherecookies.handler
-  (:require [aretherecookies.db :refer [query-food-items
+  (:require [aretherecookies.db :refer [insert-quantity
-                                        insert-quantity
+                                        select-quantities
-                                        create-food-item
-                                        get-images
-                                        add-image
                                         get-faves
-                                        add-faves
+                                        add-fave
-                                        delete-faves]]
+                                        delete-faves
-            [aretherecookies.parsers :refer [parse-special-types
+                                        search-products]]
-                                             parse-location]]
             [aretherecookies.helpers :refer [safe-json]]
             [aretherecookies.search.heb :refer [memoized-search-heb]]
-            [clojure.data.json :as json]
-            [aretherecookies.aws :refer [put-s3
-                                         build-s3-url]]
-            [clojure.string :as str]
             [buddy.auth :refer [authenticated?
-                                throw-unauthorized]]
+                                throw-unauthorized]]))
-            [net.cgrand.enlive-html :as html])
-  (:import (java.util UUID)))
 
 
 (defn bad-request
@@ -27,140 +17,66 @@
   {:status 400 :body (safe-json body)})
 
 
-(defn update-key-to-s3
+(defn quantity-add-handler
-  "return a map with key updated to point to s3"
+  "insert a quantity for a product"
-  ([key]
-   (fn [map]
-     (update map key build-s3-url)))
-  ([key map]
-   (update map key build-s3-url)))
-
-
-(defn food-items-handler
-  "return a list of food items for a given filter and location"
   [req]
-  (println "/fooditems ---->" (:body req))
+  (let [{{productId :productId quantity :quantity} :body} req]
-  (let [{body :body} req]
+    (println "/quantity POST ---->" productId quantity)
-    (safe-json
+    (if-not (authenticated? req)
-     (hash-map
+      (throw-unauthorized)
-      :filter (:filter body)
+      (safe-json (insert-quantity {:productId productId :quantity quantity})))))
-      :fooditems (->>
-                  (query-food-items body)
-                  (map parse-location)
-                  (map (update-key-to-s3 :thumbimage)))))))
 
 
-(defn quantity-handler
+(defn quantity-get-handler
-  "get the latest quantity for a food item"
+  "get the quantity for a product"
   [req]
-  (let [{{foodItemId :foodItemId quantity :quantity} :body} req]
+  (let [{{productIds :productIds} :body} req]
-    ;(if-not (authenticated? req) (throw-unauthorized))
+    (println "/quantity GET ---->" productIds)
-    (println "/quantity ---->" foodItemId quantity)
+    (if-not (authenticated? req)
-    (json/write-str
+      (throw-unauthorized)
-     (insert-quantity {:foodItemId foodItemId :quantity quantity})
+      (safe-json (select-quantities {:productIds productIds})))))
-     :value-fn parse-special-types)))
-
-(defn key-found?
-  [obj key]
-  (not (str/blank? (str (get obj key)))))
-
-(defn get-missing-keys
-  "returns a list of required keys which were not found on the given food item"
-  [foodItem]
-  (remove
-   #(key-found? foodItem %)
-   [:name :placeId :latitude :longitude :category :quantity]))
-
-
-(defn add-food-item-handler
-  "validate food item fields from request and insert into database returning newly created item"
-  [req]
-  (println "/addfooditem ---->" (:body req))
-  ;(if-not (authenticated? req) (throw-unauthorized))
-  (let [food-item (:body req) missing-keys (get-missing-keys food-item)]
-    (if (> (count missing-keys) 0)
-      (bad-request {"missingkeys" missing-keys})
-      (as->
-       (create-food-item food-item) %
-        (parse-location %)
-        (safe-json %)))))
-
-
-(defn get-images-handler
-  "return all images for a given foodItemId"
-  [req]
-  (let [foodItemId (get-in req [:params :foodItemId])]
-    (->>
-     (get-images foodItemId)
-     (map #(update % :url build-s3-url))
-     safe-json)))
-
-
-(defn add-image-handler
-  "given foodItemId from route and photo from multipart form post uploads image into s3 and adds URL into food item record"
-  [req]
-  ; (if-not (authenticated? req) (throw-unauthorized))
-  (let [username (get-in req [:params :username] "Bart Akeley")
-        foodItemId (get-in req [:params :foodItemId])
-        image (get-in req [:params :photo :tempfile])
-        img-name (str foodItemId "/" (UUID/randomUUID) ".jpg")]
-    (println "/addimage ---->" img-name)
-    (put-s3 img-name image)
-    (->>
-     (add-image {:food_item_id foodItemId
-                 :filename img-name
-                 :username username})
-     first
-     (update-key-to-s3 :url)
-     safe-json)))
 
 
 (defn faves-get-handler
   "return all the faves for a given username"
   [req]
-  (if-not (authenticated? req) (throw-unauthorized))
+  (if-not (authenticated? req)
+    (throw-unauthorized)
     (let [email (get-in req [:body :email])]
       (println "/faves/ ----> POST " email)
-    (->>  (get-faves email) safe-json)))
+      (->>  (get-faves email) safe-json))))
 
 
 (defn faves-put-handler
   "add a food item to faves for a user"
   [req]
-  (if-not (authenticated? req) (throw-unauthorized))
+  (if-not (authenticated? req)
+    (throw-unauthorized)
     (let [email (get-in req [:body :email])
-        foodItemIds (get-in req [:body :foodItemIds])]
+          productId (get-in req [:body :productId])
-    (println "/fave/" email "----> PUT" foodItemIds)
+          photoUrl (get-in req [:body :photoUrl])]
-    (->>  (add-faves email foodItemIds) safe-json)))
+      (println "/fave/" email "----> PUT" productId photoUrl)
+      (->> (add-fave email productId photoUrl) safe-json))))
 
 
 (defn faves-delete-handler
   "add a food item to faves for a user"
   [req]
-  (if-not (authenticated? req) (throw-unauthorized))
+  (if-not (authenticated? req)
+    (throw-unauthorized)
     (let [email (get-in req [:body :email])
-        foodItemIds (get-in req [:body :foodItemIds])]
+          productIds (get-in req [:body :productIds])]
-    (println "/faves/" email "----> DELETE" foodItemIds)
+      (println "/faves/" email "----> DELETE" productIds)
-    (->>  (delete-faves email foodItemIds) safe-json)))
+      (->>  (delete-faves email productIds) safe-json))))
 
 
-(defn heb-product-to-food-item
+(defn product-search-handler
-  [html-node]
+  "query cached search results and schedule a cache refresh if needed"
-  (let [image-url (:data-src (:attrs (first (html/select html-node [:img]))))
-        content-data (json/read-str (first (:content (first (html/select html-node [:script])))))]
-    {:id (str "HEB:" (get content-data "id"))
-     :name (get content-data "name")
-     :thumbimage image-url
-     :placeType "HEB"
-     :category (get content-data "category")
-     :variant (get content-data "variant")}))
-
-
-(defn heb-search-handler
-  "passes a search text to heb and parses html response"
   [req]
   (let [search (get-in req [:params :search])]
-    (println (str "/heb/" search))
+    (println (str "/search/" search))
     (if-not (empty? search)
-      (safe-json {:products (memoized-search-heb search)})
+      (let [products (search-products search)]
+        (queue-search search)
+        (safe-json {:products products})
       (safe-json {:products []}))))

src/aretherecookies/queries.sql

@@ -1,109 +1,53 @@
--- :name select-food-items :? :raw
-SELECT
-    f.id AS id,
-    f.name AS name,
-    f.place_id AS placeid,
-    f.category AS category,
-    ST_AsGeoJSON(f.loc) AS location,
-    ST_Distance(f.loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography) / 1609 AS distance,
-    q.quantity AS quantity,
-    q.date AS lastupdated,
-    (select filename from images i where f.id = i.food_item_id order by date asc limit 1) as thumbimage
-FROM food_items f 
-LEFT OUTER JOIN latest_quantities q
-ON f.id = q.food_item_id
-:snip:where
-:snip:order
-
--- :snip has-category
-WHERE 
-ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609) 
-AND 
-category IN (:i*:categories)
-
--- :snip within-radius
-WHERE ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609)
-
--- :snip food-items-where-clause
-WHERE
-ST_DWithin(loc, ST_SetSRID(ST_Point(:lng, :lat), 4326)::geography, :dist * 1609)
---~ (when (seq (:categories params)) "AND category IN (:i*:categories)")
---~ (when (not-empty (:search params)) "AND UPPER(name) like UPPER(:v:search)")
-
--- :snip by-distance
-ORDER BY distance ASC
-
--- :snip by-last-updated
-ORDER BY lastUpdated DESC
-
--- :snip by-quantity
-ORDER BY quantity DESC
-
 -- :name insert-quantity-query
-INSERT INTO quantities (food_item_id, quantity, date) 
+INSERT INTO quantities (product_id, quantity, date)
-VALUES (:v:food_item_id::uuid, :v:quantity::quantity, now())
+VALUES (:v:product_id, :v:quantity::quantity, now())
 RETURNING *
 
--- :name select-latest-quantity-query
+-- :name select-quantities-query
-SELECT food_item_id, quantity, date AS updated 
+SELECT product_id, quantity, date
-FROM quantities 
+FROM latest_quantities
-WHERE food_item_id=:i:food_item_id 
+WHERE product_id IN (:v*:product_ids)
-ORDER BY date DESC LIMIT 1
 
--- :name insert-food-item
+-- :name select-from-faves-query
-INSERT INTO food_items (id, name, place_id, category, loc)
+SELECT product_id, photo_url, date
-  VALUES (
-		uuid_generate_v4(),
-		:v:name,
-		:v:placeId,
-		:v:category::category,
-		ST_SetSRID(ST_Point(:longitude, :latitude), 4326)::geography
-	)
-RETURNING
-    id AS id,
-    name AS name,
-    place_id AS placeid,
-    category AS category,
-    ST_AsGeoJSON(loc) AS location,
-    ST_Distance(loc, ST_SetSRID(ST_Point(:longitude, :latitude), 4326)::geography) / 1609 AS distance
-
--- :name select-images
-SELECT
-    filename as url,
-    date,
-    username,
-    food_item_id
-FROM images
-WHERE food_item_id=:v:foodItemId::uuid
-
--- :name insert-image
-INSERT INTO images (filename, username, date, food_item_id)
-    VALUES (
-        :v:filename,
-        :v:username,
-        current_timestamp,
-        :v:food_item_id::uuid
-    )
-RETURNING
-    filename as url,
-    date,
-    username,
-    food_item_id
-
-
--- :name select-from-faves
-SELECT food_item_id, date
 FROM faves 
 WHERE username=:v:username
 ORDER BY date DESC
 
-
+-- :name insert-into-faves-query
--- :name insert-into-faves
+INSERT INTO faves (product_id, username, photo_url, date)
-INSERT INTO faves (food_item_id, username, date)
+VALUES (:v:product_id, :v:username, current_timestamp)
-VALUES (:v:food_item_id::uuid, :v:username, current_timestamp)
 ON CONFLICT DO NOTHING
-RETURNING food_item_id, date
+RETURNING product_id, photo_url, date
 
+-- :name delete-from-faves-query
+DELETE FROM faves
+WHERE username=:v:username
+AND product_id IN (:v*:product_ids)
+RETURNING product_id, date
 
--- :name delete-from-faves
+-- :name search-products-by-term
-DELETE FROM faves WHERE username=:v:username AND food_item_id IN (:v*:food_item_ids::uuid) RETURNING food_item_id, date
+SELECT product.* FROM product
+INNER JOIN search_term_product on search_term_product=product.id
+INNER JOIN search_term ON search_term_product.term=search_term.term
+WHERE search_term.term=:v:search
+OR dmetaphone(search_term.term)=dmetaphone(:v:search)
+OR dmetaphone_alt(search_term.term)=dmetaphone(:v:search)
+
+-- :name select-search-term
+SELECT * FROM search_term WHERE term=:v:term
+
+-- :name insert-into-product
+INSERT INTO product (id, provider_type, name, photo) VALUES :t*:products
+ON CONFLICT DO NOTHING
+RETURNING id;
+
+-- :name insert-into-search-term
+INSERT INTO search_term (term, last_queried) VALUES (:v:term, now())
+ON CONFLICT (term) DO UPDATE SET last_queried=now()
+RETURNING last_queried;
+
+-- :name insert-into-search-term-product
+INSERT INTO search_term_product (term, product_id) VALUES :t*:pairs
+ON CONFLICT DO NOTHING
+RETURNING term, product_id;

src/aretherecookies/search/heb.clj

@@ -10,7 +10,7 @@
         content-data (json/read-str (first (:content (first (html/select html-node [:script])))))]
     {:id (str "HEB:" (get content-data "id"))
      :name (get content-data "name")
-     :thumbimage image-url
+     :photo image-url
      :placeType "HEB"
      :category (get content-data "category")
      :variant (get content-data "variant")}))

src/aretherecookies/search/search.clj

@@ -0,0 +1,31 @@
+(ns aretherecookies.search.search
+  (:require [clojure.core.async :as async :refer [chan, sliding-buffer, go, >!, <!]]
+            [aretherecookies.db :refer [get-search-term, add-products, update-search-term]]
+            [aretherecookies.search.heb :refer [search-heb]]))
+
+
+(def searches-chan (chan (sliding-buffer 500)))
+
+
+(defn queue-search
+  ""
+  [term]
+  ;; TODO change empty check to a datetime check against some staleness threshold
+  (go (if (empty? (:last_queried (first (get-search-term term)))) (>! searches-chan term) nil)))
+
+
+(defn search-all-sources
+  ""
+  [term]
+  (let [heb-result (go (search-heb term))]
+    (update-search-term term)
+    (async/map #(add-products term %) [heb-result])))
+
+
+(defn init-search
+  ""
+  []
+  (go (while true
+        (try (search-all-sources (<! searches-chan))
+             (catch Exception e (println "CAUGHT:" e))))))
+