implemented some historical data scanning

Aux-Addon.toc

@@ -2,7 +2,7 @@
 ## Title: Aux
 ## Notes: A lightweight addon designed to help manage auctions
 ## SavedVariablesPerCharacter: AUX_SELL_SHORTCUT, AUX_BUY_SHORTCUT, AUX_OPEN_SELL, AUX_OPEN_BUY, AUX_INSTANT_BUYOUT, AUX_AUCTION_DURATION
-## SavedVariables: auxSellEntries
+## SavedVariables: aux_history, aux_snapshot, aux_post_parameters
 core.xml
 control.xml
 util.lua
@@ -16,6 +16,7 @@ completion.xml
 
 sell.xml
 buy.xml
+history.xml
 
 options.xml
 about.xml

buy.lua

@@ -103,7 +103,7 @@ Aux.buy.modes = {
             },
             {
                 title = 'Buy/ea',
-                width = 110,
+                width = 93,
                 comparator = function(group1, group2) return Aux.util.compare(group1[1].buyout_price_per_unit, group2[1].buyout_price_per_unit, Aux.util.GT) end,
                 cell_initializer = Aux.sheet.default_cell_initializer('RIGHT'),
                 cell_setter = function(cell, group)
@@ -113,7 +113,7 @@ Aux.buy.modes = {
             },
             {
                 title = 'Buy',
-                width = 110,
+                width = 93,
                 comparator = function(group1, group2) return Aux.util.compare(group1[1].buyout_price, group2[1].buyout_price, Aux.util.GT) end,
                 cell_initializer = Aux.sheet.default_cell_initializer('RIGHT'),
                 cell_setter = function(cell, group)
@@ -121,6 +121,23 @@ Aux.buy.modes = {
                     group_alpha_setter(cell, group)
                 end,
             },
+            {
+                title = 'Pct',
+                width = 30,
+                comparator = function(group1, group2)
+                    local market_price1 = Aux.history.get_market_price(group1[1].key)
+                    local market_price2 = Aux.history.get_market_price(group2[1].key)
+                    local factor1 = market_price1 and group1[1].buyout_price_per_unit / market_price1
+                    local factor2 = market_price2 and group2[1].buyout_price_per_unit / market_price2
+                    return Aux.util.compare(factor1, factor2, Aux.util.GT)
+                end,
+                cell_initializer = Aux.sheet.default_cell_initializer('RIGHT'),
+                cell_setter = function(cell, group)
+                    local market_price = Aux.history.get_market_price(group[1].key)
+                    cell.text:SetText(market_price and ceil(100 / group[1].buyout_price_per_unit * market_price)..'%' or 'N/A')
+                    group_alpha_setter(cell, group)
+                end,
+            },
         },
         sort_order = {{column = 2, order = 'ascending' }, {column = 4, order = 'ascending'}},
 	},
@@ -433,8 +450,6 @@ function Aux.buy.on_close()
 end
 
 function Aux.buy.on_open()
-    AuxBuySearchButton:Show()
-    AuxBuyStopButton:Hide()
 	update_sheet()
 end
 
@@ -722,7 +737,8 @@ function create_auction_record(auction_item, current_page)
     end
 
     return {
-            signature = Aux.auction_signature(auction_item.hyperlink, stack_size, bid, auction_item.buyout_price),
+            key = Aux.auction_signature(auction_item.hyperlink, stack_size, bid, auction_item.buyout_price),
+            signature = Aux.auction_signature(auction_item.hyperlink, stack_size, bid, auction_item.buyout_price), -- TODO remove
 
             name = auction_item.name,
             level = auction_item.level,

core.lua

@@ -171,6 +171,7 @@ function Aux.on_tab_click(index)
 
     AuxSellFrame:Hide()
     AuxBuyFrame:Hide()
+    AuxHistoryFrame:Hide()
 
     if index == 1 then
         AuxBuyFrame:Show()
@@ -178,6 +179,9 @@ function Aux.on_tab_click(index)
     elseif index == 2 then
         AuxSellFrame:Show()
         Aux.sell.on_open()
+    elseif index == 5 then
+        AuxHistoryFrame:Show()
+        Aux.history.on_open()
     end
 
 end

history.lua

@@ -0,0 +1,383 @@
+local MAX_HISTORY_SIZE = 500
+local MIN_SEEN = 2
+local UNDERCUT_FACTOR = 0.2
+
+Aux.history = {}
+
+aux_snapshot = {
+    signatures = {},
+    records = {},
+}
+
+aux_history = {}
+
+local cache
+
+local get_history, set_history, process_auction, flush_cache, create_balanced_list
+local get_market_price, get_usable_median, get_historical_median, get_snapshot_median, get_median, get_percentile
+
+function Aux.history.on_close()
+
+end
+
+function Aux.history.on_open()
+
+end
+
+function Aux.history.start_scan()
+
+    if not AuxHistoryScanButton:IsVisible() then
+        return
+    end
+
+    AuxHistoryScanButton:Hide()
+    AuxHistoryStopButton:Show()
+
+    cache = {}
+
+    Aux.log('Scanning auctions ...')
+    Aux.scan.start{
+        query = {},
+        page = 0,
+        on_page_loaded = function(page, total_pages)
+            Aux.log('Scanning page '..(page+1)..' out of '..total_pages..' ...')
+        end,
+        on_read_auction = function(i)
+            process_auction(i)
+        end,
+        on_complete = function()
+            flush_cache()
+
+            AuxHistoryStopButton:Hide()
+            AuxHistoryScanButton:Show()
+        end,
+        on_abort = function()
+            flush_cache()
+
+            AuxHistoryStopButton:Hide()
+            AuxHistoryScanButton:Show()
+        end,
+        next_page = function(page, total_pages)
+            if AuxBuyAllPagesCheckButton:GetChecked() then
+                local last_page = max(total_pages - 1, 0)
+                if page < last_page then
+                    return page + 1
+                end
+            end
+        end,
+    }
+end
+
+function Aux.history.stop_scan()
+    Aux.scan.abort()
+end
+
+function get_history(key)
+    local history_key = GetCVar('realmName')
+    aux_history[history_key] = aux_history[history_key] or {}
+    aux_history[history_key][key] = aux_history[history_key][key] or {}
+    return aux_history[history_key][key]
+end
+
+function set_history(key, history)
+    local history_key = GetCVar('realmName')
+    aux_history[history_key] = aux_history[history_key] or {}
+    aux_history[history_key][key] = history
+end
+
+function process_auction(index)
+    local auction_info = Aux.info.auction_item(index)
+    local buyout_price = Aux.util.safe_index{auction_info, 'buyout_price'}
+    if buyout_price and buyout_price > 0 then
+        local aux_quantity = auction_info.charges or auction_info.count
+        local price = ceil(buyout_price / aux_quantity)
+        local key = auction_info.item_signature
+        cache[Aux.info.auction_signature(index)] = {key=key, price=price}
+    end
+end
+
+function flush_cache()
+
+    -- local timestamp = time()
+
+    local new_history = {}
+
+    aux_snapshot.records = {}
+
+    for sig, auction_data in pairs(cache) do
+        local key = auction_data.key
+        aux_snapshot.records[key] = aux_snapshot.records[key] or {}
+        tinsert(aux_snapshot.records[key], auction_data.price)
+        snipe.log(getn(aux_snapshot.records[key]))
+
+        if not Aux.util.set_contains(aux_snapshot.signatures, sig) then
+
+            Aux.util.set_add(aux_snapshot.signatures, sig)
+
+            if not new_history[key] then
+                new_history[key] = create_balanced_list(MAX_HISTORY_SIZE)
+                new_history[key].add_all(get_history(key))
+            end
+            new_history[key].add(auction_data.price)
+        end
+    end
+
+    for key, history in new_history do
+        set_history(key, history.values())
+    end
+
+    for sig, _ in ipairs(aux_snapshot.signatures) do
+        if not cache[sig] then
+            Aux.util.set_remove(aux_snapshot.signatures, sig)
+        end
+    end
+end
+
+--function store_median_list(list)
+--    local hist = ''
+--    local function grow_list(last, n)
+--        if n == 1 then
+--            hist = hist == '' and last or string.format('%s:%d', hist, last)
+--        elseif n > 1 then
+--            hist = hist == '' and string.format('%dx%d', last, n) or string.format('%s:%dx%d', hist, last, n)
+--        end
+--    end
+--    local n = 0
+--    local last = 0
+--    for i, price in pairs(list) do
+--        if i == 1 then
+--            last = price
+--        elseif price ~= last then
+--            grow_list(last, n)
+--            last = price
+--            n = 0
+--        end
+--        n = n + 1
+--    end
+--    grow_list(last, n)
+--    return hist
+--end
+--
+--function load_median_list(str)
+--    local splut = {}
+--    for x, c in string.gfind(str, '([^%:]*)(%:?)') do
+--        local _, _, y, n = string.find(x, '(%d*)x(%d*)')
+--        if y == nil then
+--            table.insert(splut, tonumber(x))
+--        else
+--            for i = 1,n do
+--                table.insert(splut, tonumber(y))
+--            end
+--        end
+--        if c == '' then break end
+--    end
+--    return splut
+--end
+
+function Aux.history.get_market_price(key)
+    local price
+
+    local median = get_usable_median(key)
+--    local avgMin, avgBuy, avgBid, bidPct, buyPct, avgQty, meanCount = getMeans(key, realm)
+
+    -- assign the best common buyout
+    if median and median > 0 then
+        price = median
+--    elseif meanCount and meanCount > 0 then
+--        -- if a usable median does not exist, use the average buyout instead
+--        price = avgBuy;
+    end
+
+--    local playerMade, skill, level = Auctioneer.Core.IsPlayerMade(key);
+--    if Auctioneer.Core.Constants.BidBasedCategories[Auctioneer.Core.GetItemCategory(key)] and not (playerMade and level < 250 and commonBuyout < 100000) then
+--        -- returns bibasedSellablePrice for bidbaseditems, playermade items or if the buyoutprice is not present or less than 10g
+--        return getBidBasedSellablePrice(key, realm, avgMin,avgBuy,avgBid,bidPct,buyPct,avgQty,seenCount)
+--    end
+
+    return price
+end
+
+function get_usable_median(key)
+
+    local snapshot_median, snapshot_count = get_snapshot_median(key)
+    local historical_median, historical_count = get_historical_median(key)
+
+    if snapshot_count >= MIN_SEEN and (historical_count < snapshot_count or snapshot_median < 1.2 * historical_median) then
+        return snapshot_median, snapshot_count
+    elseif historical_count >= MIN_SEEN then
+        return historical_median, historical_count
+    end
+end
+
+function get_snapshot_median(key)
+    local median, count
+
+--    if AuctionConfig.stats and AuctionConfig.stats.snapmed and AuctionConfig.stats.snapmed[auctKey] then
+--        median = AuctionConfig.stats.snapmed[auctKey][key]
+--        count = AuctionConfig.stats.snapcount[auctKey][key]
+--    end
+
+    if (not median) or (not count) then
+
+        local snapshot_prices = aux_snapshot.records[key]
+        median, count = get_median(snapshot_prices or {})
+
+--        Auctioneer.Storage.SetSnapMed(auctKey, key, median, count)
+    end
+
+    return median, count
+end
+
+function get_historical_median(key)
+    local median, count
+
+--    if AuctionConfig.stats and AuctionConfig.stats.histmed and AuctionConfig.stats.histmed[auctKey] then
+--        median = AuctionConfig.stats.histmed[auctKey][key]
+--        count = AuctionConfig.stats.histcount[auctKey][key]
+--    end
+
+    if (not median) or (not count) then
+
+        local historical_prices = get_history(key)
+        median, count = get_median(historical_prices or {})
+
+--        Auctioneer.Storage.SetHistMed(auctKey, key, median, count)
+    end
+
+    return median, count
+end
+
+function get_median(values)
+    return get_percentile(values, 0.5)
+end
+
+-- Return weighted average percentile such that returned value
+-- is larger than or equal to (100*pct)% of the table values
+-- 0 <= pct <= 1
+function get_percentile(values, pct)
+
+    local _percentile = function(sorted_values, pct, first, last)
+        local f = (last - first) * pct + first
+        local i1, i2 = floor(f), ceil(f)
+        f = f - i1
+
+        return sorted_values[i1] * (1 - f) + sorted_values[i2] * f
+    end
+
+    local n = getn(values)
+
+    if n == 0 then
+        return 0, 0 -- if there is an empty table, returns median = 0, count = 0
+    elseif n == 1 then
+        return tonumber(values[1]), 1
+    end
+
+    -- The following calculations require a sorted table
+    table.sort(values)
+
+    -- Skip IQR calculations if table is too small to have outliers
+    if n <= 4 then
+        return _percentile(values, pct, 1, n), n
+    end
+
+    --  REWORK by Karavirs to use IQR*1.5 to ignore outliers
+    -- q1 is median 1st quartile q2 is median of set q3 is median of 3rd quartile iqr is q3 - q1
+    local q1 = _percentile(values, 0.25, 1, n)
+    local q3 = _percentile(values, 0.75, 1, n)
+
+    local iqr = (q3 - q1) * 1.5
+    local iqlow, iqhigh = q1 - iqr, q3 + iqr
+
+    -- Find first and last index to include in median calculation
+    local first, last = 1, n
+
+    -- Skip low outliers
+    while values[first] < iqlow do
+        first = first + 1
+    end
+
+    -- Skip high outliers
+    while values[last] > iqhigh do
+        last = last - 1
+    end
+
+    return _percentile(values, pct, first, last), last - first + 1
+end
+
+function create_balanced_list(max_size, cmp)
+    local self = {}
+
+    local values = {}
+
+    cmp = cmp or Aux.util.compare
+
+    function self.add(value)
+
+        local left = 1
+        local right = getn(values)
+        local middle_value
+        local middle
+
+        local destination
+        while left <= right do
+            middle = floor((right - left) / 2) + left
+            middle_value = values[middle]
+            if cmp(value, middle_value) == Aux.util.LT then
+                right = middle - 1
+            elseif cmp(value, middle_value) == Aux.util.GT then
+                left = middle + 1
+            else
+                destination = middle
+                break
+            end
+        end
+        destination = destination or left
+
+        tinsert(values, destination, value)
+
+        if max_size and getn(values) > max_size then
+            if destination <= floor(max_size / 2) + 1 then
+                tremove(values)
+            else
+                tremove(values, 1)
+            end
+        end
+    end
+
+    function self.add_all(array)
+        self.clear()
+        for _, value in ipairs(array) do
+            self.add(value)
+        end
+    end
+
+    function self.clear()
+        values = {}
+    end
+
+    function self.values()
+        local result = {}
+        for _, value in ipairs(values) do
+            tinsert(result, value)
+        end
+        return result
+    end
+
+    function self.size()
+        return getn(values)
+    end
+
+    function self.get(index)
+        return values[index]
+    end
+
+    function self.max_size()
+        return max_size
+    end
+
+    return self
+end
+
+function Aux.history.get_price_suggestion(key, quantity)
+    return get_market_price(key) * quantity * UNDERCUT_FACTOR or 0
+end

history.xml

@@ -0,0 +1,36 @@
+<Ui xsi:schemaLocation="http://www.blizzard.com/wow/ui/">
+
+    <Script file="history.lua"/>
+
+    <Frame>
+        <Scripts>
+            <OnUpdate>
+                Aux.buy.onupdate()
+            </OnUpdate>
+        </Scripts>
+    </Frame>
+
+    <Frame name="AuxHistoryFrame" hidden="true" parent="AuxFrame" setAllPoints="true">
+        <Frames>
+            <Button name="AuxHistoryScanButton" inherits="UIPanelButtonTemplate" text="Scan">
+                <Size><AbsDimension x="90" y="26"/></Size>
+                <Anchors><Anchor point="RIGHT" relativePoint="LEFT" relativeTo="AuxCloseButtonBottom"><Offset><AbsDimension x="-25" y="0"/></Offset></Anchor></Anchors>
+                <Scripts>
+                    <OnClick>
+                        Aux.history.start_scan()
+                    </OnClick>
+                </Scripts>
+            </Button>
+
+            <Button name="AuxHistoryStopButton" inherits="UIPanelButtonTemplate" text="Stop" hidden="true">
+                <Size><AbsDimension x="90" y="26"/></Size>
+                <Anchors><Anchor point="RIGHT" relativePoint="LEFT" relativeTo="AuxCloseButtonBottom"><Offset><AbsDimension x="-25" y="0"/></Offset></Anchor></Anchors>
+                <Scripts>
+                    <OnClick>
+                        Aux.history.stop_scan()
+                    </OnClick>
+                </Scripts>
+            </Button>
+        </Frames>
+    </Frame>
+</Ui>

info.lua

@@ -53,13 +53,14 @@ function Aux.info.auction_sell_item()
 end
 
 function Aux.info.auction_item(index)
-	local hyperlink = GetAuctionItemLink("list", index)
+	local hyperlink = GetAuctionItemLink('list', index)
 	
 	if not hyperlink then
 		return
 	end
 	
 	local auction_item = hyperlink_item(hyperlink)
+    auction_item.item_signature = item_signature(hyperlink)
 	
 	local name, texture, count, quality, usable, level, min_bid, min_increment, buyout_price, current_bid, high_bidder, owner, sale_status, id = GetAuctionItemInfo("list", index)
 	local duration = GetAuctionItemTimeLeft("list", index)
@@ -153,6 +154,38 @@ function item_charges(tooltip)
 	end
 end	
 
+function Aux.info.auction(args)
+    local index = tremove(args, 1)
+
+    local key_set = Aux.util.array_to_set(args)
+end
+
+-- not unique!
+function Aux.info.auction_signature(index)
+    -- owner not used because waiting for it would slow down the scan too much
+    local _, _, count, _, _, _, min_bid, _, buyout_price, _, _, owner = GetAuctionItemInfo('list', index)
+    local hyperlink = GetAuctionItemLink('list', index)
+    local item_id, suffix_id, enchant_id, unique_id = parse_hyperlink(hyperlink)
+    return string.format(
+        '%s:%s:%s:%s:%s:%s:%s',
+        item_id or 0,
+        suffix_id or 0,
+        enchant_id or 0,
+        unique_id or 0,
+        count or 0,
+        min_bid or 0,
+        buyout_price or 0
+    )
+end
+
+function Aux.info.break_auction_signature(signature)
+    local parts = strfind(signature, '^(%d+):(%d+):(%d+):(%d+):(%d+):(%d+):(%d+)$')
+    for i=1,2 do
+        tremove(parts, 1)
+    end
+    return unpack(parts)
+end
+
 function item_signature(hyperlink)
     local item_id, suffix_id = parse_hyperlink(hyperlink)
     return item_id..':'..suffix_id

sell.lua

@@ -1,6 +1,6 @@
 Aux.sell = {}
 
-auxSellEntries = {} -- persisted
+local auxSellEntries = {}
 
 local inventory_data
 
@@ -10,7 +10,7 @@ local current_auction
 
 local set_auction, update_auction_listing, update_inventory_listing, record_auction, undercut, item_class_index, item_subclass_index, report, select_entry, update_recommendation, refresh_entries, auction_candidates, charge_classes, get_stack_size_slider_value
 
-local LIVE, HISTORICAL = 1, 2
+local LIVE, HISTORICAL, FIXED = 1, 2, 3
 
 Aux.sell.inventory_listing_config = {
     on_cell_click = function (sheet, row_index, column_index)
@@ -381,46 +381,25 @@ function update_recommendation()
         -- TODO neutral AH deposit formula
 		MoneyFrame_Update('AuxSellParametersDepositMoneyFrame', floor(current_auction.unit_vendor_price * 0.05 * (current_auction.charges and 1 or get_stack_size_slider_value())) * AuxSellStackCountSlider:GetValue() * AuctionFrameAuctions.duration / 120)
 		
-		if auxSellEntries[current_auction.name] and auxSellEntries[current_auction.name].selected then
+		if UIDropDownMenu_GetSelectedValue(AuxSellParametersStrategyDropDown) == LIVE and auxSellEntries[current_auction.name] and auxSellEntries[current_auction.name].selected then
 			if not auxSellEntries[current_auction.name].created or GetTime() - auxSellEntries[current_auction.name].created > 1800 then
                 AuxSellParametersStrategyDropDownStaleWarning:SetText('Stale data!') -- data older than half an hour marked as stale
 			end
 		
-			local newBuyoutPrice = auxSellEntries[current_auction.name].selected.unit_buyout_price * get_stack_size_slider_value()
+			local new_buyout_price = auxSellEntries[current_auction.name].selected.unit_buyout_price * get_stack_size_slider_value()
 
 			if auxSellEntries[current_auction.name].selected.yours == 0 then
-				newBuyoutPrice = undercut(newBuyoutPrice)
+				new_buyout_price = undercut(new_buyout_price)
 			end
 			
-			local newStartPrice = newBuyoutPrice * 0.95
+			local new_start_price = new_buyout_price * 0.95
 			
---			Aux_ShowElems(Aux.tabs.sell.recommendationElements)
-			
---			AuxRecommendText:SetText("Recommended Buyout Price")
---			AuxRecommendPerStackText:SetText("for a stack of "..get_stack_size_slider_value())
-			
-
---			AuxRecommendItemTex:SetNormalTexture(current_auction.texture)
---			if AuxSellStackSize:GetNumber() > 1 then
---				AuxRecommendItemTexCount:SetText(get_stack_size_slider_value())
---				AuxRecommendItemTexCount:Show()
---			else
---				AuxRecommendItemTexCount:Hide()
---			end
-
---			MoneyFrame_Update("AuxRecommendPerItemPrice",  Aux_Round(get_stack_size_slider_value() > 0 and newBuyoutPrice / get_stack_size_slider_value() or 0))
---			MoneyFrame_Update("AuxRecommendPerStackPrice", Aux_Round(newBuyoutPrice))
-			
-			MoneyInputFrame_SetCopper(AuxSellParametersBuyoutPrice, Aux_Round(newBuyoutPrice))
-			MoneyInputFrame_SetCopper(AuxSellParametersStartPrice, Aux_Round(newStartPrice))
-			
---			if auxSellEntries[current_auction.name].selected.stack_size == auxSellEntries[current_auction.name][1].stack_size and auxSellEntries[current_auction.name].selected.buyout_price == auxSellEntries[current_auction.name][1].buyout_price then
---				AuxRecommendBasisText:SetText("(based on cheapest)")
---			elseif bestPriceOurStackSize and auxSellEntries[current_auction.name].selected.stack_size == bestPriceOurStackSize.stack_size and auxSellEntries[current_auction.name].selected.buyout_price == bestPriceOurStackSize.buyout_price then
---				AuxRecommendBasisText:SetText("(based on cheapest stack of the same size)")
---			else
---				AuxRecommendBasisText:SetText("(based on auction selected below)")
---			end
+			MoneyInputFrame_SetCopper(AuxSellParametersBuyoutPrice, Aux_Round(new_buyout_price))
+			MoneyInputFrame_SetCopper(AuxSellParametersStartPrice, Aux_Round(new_start_price))
+        elseif UIDropDownMenu_GetSelectedValue(AuxSellParametersStrategyDropDown) == HISTORICAL then
+            local market_price = Aux.history.get_price_suggestion(current_auction.key, current_auction.aux_quantity)
+            MoneyInputFrame_SetCopper(AuxSellParametersBuyoutPrice, Aux_Round(market_price * 0.95))
+            MoneyInputFrame_SetCopper(AuxSellParametersStartPrice, market_price)
         else
             MoneyInputFrame_SetCopper(AuxSellParametersBuyoutPrice, 0)
             MoneyInputFrame_SetCopper(AuxSellParametersStartPrice, 0)
@@ -526,6 +505,7 @@ function update_inventory_data()
                     if not auction_candidate_map[item_info.item_signature] then
 
                         auction_candidate_map[item_info.item_signature] = {
+                            key = item_info.item_signature,
                             hyperlink = item_info.hyperlink,
                             name = item_info.name,
                             texture = item_info.texture,
@@ -573,7 +553,7 @@ function refresh_entries()
 		-- local class_index = item_class_index(current_auction.class)
 		-- local subclass_index = item_subclass_index(class_index, current_auction.subclass)
 
-		Aux.log('Starting scan.')
+		Aux.log('Scanning auctions ...')
 		Aux.scan.start{
 			query = {
 				name = name,
@@ -720,9 +700,15 @@ function Aux.sell.initialize_strategy_dropdown()
         func = on_click,
     }
 
+    UIDropDownMenu_AddButton{
+        text = 'Historical price',
+        value = HISTORICAL,
+        func = on_click,
+    }
+
 --    UIDropDownMenu_AddButton{
---        text = 'Historical price',
---        value = HISTORICAL,
+--        text = 'Fixed',
+--        value = FIXED,
 --        func = on_click,
 --    }
 end