From 10349381ad9151a89fa11686aca836e49370e405 Mon Sep 17 00:00:00 2001 From: Manuel Simon Hirsig Date: Sat, 16 Jan 2016 17:03:23 +0100 Subject: [PATCH] some progress on price history --- Aux-AddOn.toc | 6 +- core.lua | 29 ++++++- info.lua | 6 +- persistence.lua | 59 ++++++++------- scan_frame.lua | 83 ++++++++------------ stat_average.lua | 191 ++++++++++++++++++++++++++++++++++++----------- 6 files changed, 245 insertions(+), 129 deletions(-) diff --git a/Aux-AddOn.toc b/Aux-AddOn.toc index 119475b..51b7031 100644 --- a/Aux-AddOn.toc +++ b/Aux-AddOn.toc @@ -1,8 +1,9 @@ ## Interface: 11200 ## Title: Aux ## Notes: A lightweight addon designed to help manage auctions -## SavedVariablesPerCharacter: aux_view, AUX_SELL_SHORTCUT, AUX_BUY_SHORTCUT, AUX_OPEN_SELL, AUX_OPEN_BUY, AUX_INSTANT_BUYOUT, AUX_AUCTION_DURATION -## SavedVariables: aux_database, aux_recently_searched, aux_auctionable_items +## Dependencies: EnhTooltip, Stubby +## SavedVariablesPerCharacter: aux_recently_searched, aux_view, AUX_SELL_SHORTCUT, AUX_BUY_SHORTCUT, AUX_OPEN_SELL, AUX_OPEN_BUY, AUX_INSTANT_BUYOUT, AUX_AUCTION_DURATION +## SavedVariables: aux_database core.xml control.xml @@ -13,6 +14,7 @@ scan.lua stack.lua post.lua sheet.lua +stat_average.lua info.xml completion.xml diff --git a/core.lua b/core.lua index f62e9e8..ad8430f 100644 --- a/core.lua +++ b/core.lua @@ -30,6 +30,28 @@ function Aux_OnLoad() Aux.log('Aux v'..AuxVersion..' loaded.') Aux.loaded = true tinsert(UISpecialFrames, 'AuxFrame') + Stubby.RegisterFunctionHook('EnhTooltip.AddTooltip', 100, function(_ ,_ ,_ ,_ , link, _, count) + local item_id, suffix_id = EnhTooltip.BreakLink(link) + local item_key = item_id..':'..suffix_id + + local auction_count = unpack(Aux.stat_average.read_record(item_key)) + + if auction_count == 0 then + EnhTooltip.AddLine('Never seen at auction', nil, true) + EnhTooltip.LineColor(0.5, 0.8, 0.5) + else + EnhTooltip.AddLine('Seen '..auction_count..' '..Aux_PluralizeIf('time', auction_count)..' at auction total', nil, true) + EnhTooltip.LineColor(0.5, 0.8, 0.1) + + local unit_value = Aux.stat_average.get_mean(item_key) + if count == 1 then + EnhTooltip.AddLine(Aux.util.money_string(unit_value), nil, true) + else + EnhTooltip.AddLine(Aux.util.money_string(unit_value * count)..' ('..Aux.util.money_string(unit_value)..' ea.)', nil, true) + end + EnhTooltip.LineColor(0.1,0.8,0.5) + end + end) end function Aux_OnEvent() @@ -53,7 +75,10 @@ end function Aux_OnAddonLoaded() if string.lower(arg1) == "blizzard_auctionui" then Aux_SetupHookFunctions() - end + end + if string.lower(arg1) == "EnhTooltip" then + + end end function Aux.log_frame_load() @@ -184,7 +209,7 @@ function Aux.on_tab_click(index) Aux.sell.on_close() Aux.auctions_frame.on_close() Aux.bids_frame.on_close() - Aux.history.on_close() + Aux.history_frame.on_close() for i=1,5 do getglobal('AuxTab'..i):SetAlpha(i == index and 1 or 0.5) diff --git a/info.lua b/info.lua index a59be78..3cfd69c 100644 --- a/info.lua +++ b/info.lua @@ -93,6 +93,7 @@ function public.auction(index, type) hyperlink = hyperlink, itemstring = item_info.itemstring, item_key = item_id..':'..suffix_id, + signature = public.auction_signature(index, type), name = name, texture = texture, @@ -205,10 +206,11 @@ function private.item_charges(tooltip) end -- not unique! -function public.auction_signature(index) +function public.auction_signature(index, type) + type = type or 'list' -- owner not used because waiting for it would slow down the scan too much local _, _, count, _, _, _, min_bid, _, buyout_price, _, _, owner = GetAuctionItemInfo('list', index) - local hyperlink = GetAuctionItemLink('list', index) + local hyperlink = GetAuctionItemLink(type, index) local item_id, suffix_id, unique_id, enchant_id = private.parse_hyperlink(hyperlink) return string.format( '%s:%s:%s:%s:%s:%s:%s', diff --git a/persistence.lua b/persistence.lua index 20283a7..48478dd 100644 --- a/persistence.lua +++ b/persistence.lua @@ -6,10 +6,14 @@ Aux.persistence = public aux_database = {} function public.on_load() - perform_migration(aux_database) + private.perform_migration(aux_database) aux_database.version = DATABASE_VERSION end +function private.perform_migration() + +end + function private.get_dataset_key() local realm = GetCVar('realmName') local zone = GetMinimapZoneText() @@ -29,37 +33,38 @@ function public.load_dataset() return aux_database[dataset_key] end -function load_snapshot() - local dataset = load_dataset() +function public.load_snapshot() + local dataset = public.load_dataset() + dataset.snapshot = dataset.snapshot or {} local snapshot = Aux.util.set() snapshot.set_data(dataset.snapshot) return snapshot end -function public.load_scan_records(item_key) - local dataset = private.load_dataset() +--function public.load_scan_records(item_key) +-- local dataset = private.load_dataset() +-- +-- local serialized_records = private.deserialize(dataset[item_key] or '', ';') +-- return Aux.util.map(serialized_records, function(serialized_record) +-- local time, count, min_price, accumulated_price = private.deserialize(serialized_record, '#') +-- return { +-- time = time, +-- count = count, +-- min_price = min_price, +-- accumulated_price = accumulated_price, +-- } +-- end) +--end +-- +--function public.store_scan_record(item_key, record) +-- local dataset = private.load_dataset() +-- local serialized_record = private.serialize({ record.time, record.count, record.min_price, record.accumulated_price }, '#') +-- local serialized_records = dataset[item_key] or '' +-- serialized_records = serialized_records..(serialized_records == '' and '' or ';')..serialized_record +-- dataset[item_key] = serialized_records +--end - local serialized_records = private.deserialize(dataset[item_key] or '', ';') - return Aux.util.map(serialized_records, function(serialized_record) - local time, count, min_price, accumulated_price = private.deserialize(serialized_record, '#') - return { - time = time, - count = count, - min_price = min_price, - accumulated_price = accumulated_price, - } - end) -end - -function public.store_scan_record(item_key, record) - local dataset = private.load_dataset() - local serialized_record = private.serialize({ record.time, record.count, record.min_price, record.accumulated_price }, '#') - local serialized_records = dataset[item_key] or '' - serialized_records = serialized_records..(serialized_records == '' and '' or ';')..serialized_record - dataset[item_key] = serialized_records -end - -function private.serialize(data, separator) +function public.serialize(data, separator) local data_string = '' for i, datum in ipairs(data) do data_string = data_string..(i == 1 and '' or separator)..datum @@ -67,7 +72,7 @@ function private.serialize(data, separator) return data_string end -function private.deserialize(data_string, separator) +function public.deserialize(data_string, separator) if data_string == '' then return {} end diff --git a/scan_frame.lua b/scan_frame.lua index 8139f4d..37b0252 100644 --- a/scan_frame.lua +++ b/scan_frame.lua @@ -1,32 +1,24 @@ -local MAX_HISTORY_SIZE = 100 -local MIN_SEEN = 1 -local UNDERCUT_FACTOR = 0.2 +local private, public = {}, {} +Aux.history_frame = public -Aux.history = {} - -local process_auction, balanced_list, update_snapshot -local get_market_price, get_usable_median, get_historical_median, get_median, get_percentile - -local auction_cache - -function Aux.history.on_close() +function public.on_close() end -function Aux.history.on_open() +function public.on_open() end -function Aux.history.start_scan() +function public.start_scan() - if not AuxHistoryScanButton:IsVisible() then - return - end +-- if not AuxHistoryScanButton:IsVisible() then +-- return +-- end AuxHistoryScanButton:Hide() AuxHistoryStopButton:Show() - auction_cache = {} + private.scanned_signatures = Aux.util.set() Aux.log('Scanning auctions ...') Aux.scan.start{ @@ -36,60 +28,49 @@ function Aux.history.start_scan() Aux.log('Scanning page '..(page+1)..' out of '..total_pages..' ...') end, on_read_auction = function(auction_info) - process_auction(auction_info) + private.process_auction(auction_info) end, on_complete = function() - process_scanned_auctions() + private.update_snapshot() AuxHistoryStopButton:Hide() AuxHistoryScanButton:Show() end, on_abort = function() - process_scanned_auctions() + private.update_snapshot() AuxHistoryStopButton:Hide() AuxHistoryScanButton:Show() end, next_page = function(page, total_pages) - if AuxBuyAllPagesCheckButton:GetChecked() then - local last_page = max(total_pages - 1, 0) - if page < last_page then - return page + 1 - end + local last_page = max(total_pages - 1, 0) + if page < last_page then + return page + 1 end end, } end -function process_scanned_auctions() - local time = time() - for item_key, auctions in pairs(auction_cache) do - local min_price, accumulated_price - for _, auction in ipairs(auctions) do - min_price = min_price and min(min_price, auction.price) or auction.price - accumulated_price = (accumulated_price or 0) + auction.price - end - Aux.persistence.store_scan_record(item_key, { - time = time, - count = getn(auctions), - min_price = min_price, - accumulated_price = accumulated_price, - }) - end -end - -function Aux.history.stop_scan() +function public.stop_scan() Aux.scan.abort() end -function process_auction(auction_info) - local buyout_price = Aux.util.safe_index{auction_info, 'buyout_price'} - if buyout_price and buyout_price > 0 then - local aux_quantity = auction_info.charges or auction_info.count - local price = ceil(buyout_price / aux_quantity) - local item_key = auction_info.item_signature +function private.update_snapshot() + local snapshot = Aux.persistence.load_snapshot() + for _, sig in ipairs(snapshot.values()) do + if not private.scanned_signatures.contains(sig) then + snapshot.remove(sig) + end + end +end - auction_cache[item_key] = auction_cache[item_key] or {} - tinsert(auction_cache[item_key], { price=price }) +function private.process_auction(auction_info) + private.scanned_signatures.add(auction_info.signature) + + local snapshot = Aux.persistence.load_snapshot() + + if not snapshot.contains(auction_info.signature) then + snapshot.add(auction_info.signature) + Aux.stat_average.process_auction(auction_info) end end diff --git a/stat_average.lua b/stat_average.lua index 629ccbb..662ce78 100644 --- a/stat_average.lua +++ b/stat_average.lua @@ -2,77 +2,178 @@ local private, public = {}, {} Aux.stat_average = public private.PUSH_INTERVAL = 57600 +private.NEW_RECORD = '0:0:0:0:0:0:0:0:0:0:0' function private.load_data() local dataset = Aux.persistence.load_dataset() - dataset.stat_average_data = dataset.stat_average_data or {} + dataset.stat_average_data = dataset.stat_average_data or { next_push = time() + private.PUSH_INTERVAL, item_data = {} } return dataset.stat_average_data end -function private.get_item_key(auction_info) - return auction_info.item_id..':'..auction_info.suffix_id +function public.read_record(item_key) + local data = private.load_data() + return Aux.util.map(Aux.persistence.deserialize(data.item_data[item_key] or private.NEW_RECORD, ':'), function(value) + return tonumber(value) + end) end -function private.process_auction(auction_info) +function private.write_record(item_key, record) + local data = private.load_data() + data.item_data[item_key] = Aux.persistence.serialize(record, ':') +end + +function public.process_auction(auction_info) + if auction_info.buyout_price == 0 then return end - - local item_record = private.load_data()[get_item_key(auction_info)] or { - daily_auction_count = 0, - daily_unit_count = 0, - accumulated_buyout_price = 0, - seen_days = 0, - EMA3 = 0, - EMA7 = 0, - EMA14 = 0, - average_min_buyout = 0, - } - - item_record.daily_auction_count = item_record.daily_auction_count + 1 - item_record.daily_unit_count = item_record.daily_unit_count + auction_info.count - item_record.accumulated_buyout_price = item_record.accumulated_buyout_price + auction_info.buyout_price - item_record.daily_min_buyout_price = item_record.daily_min_buyout_price and min(item_record.daily_min_buyout_price, auction_info.buyout_price / auction_info.count) or auction_info.buyout_price / auction_info.count + + local data = private.load_data() + + if data.next_push < time() then + private.push_data() + end + + local item_record = public.read_record(auction_info.item_key) + + item_record[1] = item_record[1] + 1 -- auction count + item_record[2] = item_record[2] + 1 -- daily auction count + item_record[3] = item_record[3] + auction_info.count -- daily unit count + item_record[4] = item_record[4] + auction_info.buyout_price -- daily accumulated buyout + item_record[5] = item_record[5] ~= 0 and min(item_record[4], auction_info.buyout_price / auction_info.count) or auction_info.buyout_price / auction_info.count -- daily min buyout + + private.write_record(auction_info.item_key, item_record) +end + +function public.get_mean(item_key) + local auction_count, daily_auction_count, daily_unit_count, daily_accumulated_buyout, daily_min_buyout, seen_days, EMA3, EMA7, EMA14, average_min_buyout = unpack(public.read_record(item_key)) + + local mean = 0 + + if seen_days == 0 then + if daily_auction_count > 0 then + mean = daily_accumulated_buyout / daily_unit_count + end + elseif seen_days <= 3 then -- No EMAs before day 4 + mean = EMA3 + if daily_auction_count > 0 then + mean = (mean * seen_days + daily_accumulated_buyout / daily_unit_count) / (seen_days + 1) + end + else + -- we have 4 or more days of data, potentially enough to perform mean and stddev calculations + local count = 0 + local valueset, weightset = {}, {} + + -- include daily data if available + if daily_auction_count > 0 then + count = 1 + valueset[count] = daily_accumulated_buyout / daily_unit_count + weightset[count] = 1 + end + + -- EMA3: standard weight 3, reduced if seenDays < 6, reduced if there was daily data, but never less than 1 + local weight = 3 - count + if seen_days < 6 then + weight = seen_days - 3 + if weight > 1 then + weight = weight - count + end + end + count = count + 1 + valueset[count] = EMA3 + weightset[count] = weight + + -- EMA7: standard weight 4, reduced if seenDays < 10 + if seen_days > 6 then + count = count + 1 + valueset[count] = EMA7 + if seen_days < 10 then + weightset[count] = seen_days - 6 + else + weightset[count] = 4 + end + end + + -- EMA14: standard weight 7, reduced if seenDays < 17 + if seen_days > 10 then + count = count + 1 + valueset[count] = EMA14 + if seen_days < 17 then + weightset[count] = seen_days - 10 + else + weightset[count] = 7 + end + end + + -- we will use a weighted incremental algorithm, based on sample code by West and Knuth http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + local sumWeight, sumSquares = 0, 0 -- actually "sum of squares of differences from the (current) mean", but that's rather long for a variable name. + for i=1,count do + local value, weight = valueset[i], weightset[i] + local nextweight = weight + sumWeight + local valuediff = value - mean + local meanadjust = valuediff * weight / nextweight + mean = mean + meanadjust + sumSquares = sumSquares + sumWeight * valuediff * meanadjust + sumWeight = nextweight + end + +-- stddev = sqrt(sumSquares / sumWeight * count / (count - 1)) + end + + return mean end function private.push_data() local data = private.load_data() - for item_key, item_record in pairs(data) do + for item_key, _ in pairs(data) do - if item_record.daily_auction_count > 0 then - - local daily_average = item_record.daily_accumulated_buyout / item_record.daily_unit_count + local item_record = public.read_record(item_key) + local _, daily_auction_count, daily_unit_count, daily_accumulated_buyout, daily_min_buyout, seen_days, EMA3, EMA7, EMA14, average_min_buyout = unpack(item_record) - if item_record.seen_days < 3 then + if daily_auction_count > 0 then + + local daily_average = daily_accumulated_buyout / daily_unit_count + + if seen_days < 3 then -- for first 3 days perform plain average instead of EMAs - item_record.EMA3 = (item_record.EMA3 * item_record.seen_days + daily_average) / (item_record.seen_days + 1) - item_record.average_min_buyout = (item_record.average_min_buyout * item_record.seen_days + item_record.daily_min_buyout) / (item_record.seen_days + 1) - item_record.EMA7 = item_record.EMA3 - item_record.EMA14 = item_record.EMA3 + EMA3 = (EMA3 * seen_days + daily_average) / (seen_days + 1) + average_min_buyout = (average_min_buyout * seen_days + daily_min_buyout) / (seen_days + 1) + EMA7 = EMA3 + EMA14 = EMA3 else -- do normal EMA calculations - item_record.EMA3 = (item_record.EMA3 * 2 + daily_average) / 3) - item_record.EMA7 = (item_record.EMA7 * 6 + daily_average) / 7) - item_record.EMA14 = (item_record.EMA14 * 13 + daily_average) / 14) + EMA3 = (EMA3 * 2 + daily_average) / 3 + EMA7 = (EMA7 * 6 + daily_average) / 7 + EMA14 = (EMA14 * 13 + daily_average) / 14 - if item_record.daily_min_buyout / item_record.average_min_buyout < 0.9 then - item_record.average_min_buyout = (item_record.average_min_buyout + item_record.daily_min_buyout) / 2 + if daily_min_buyout / average_min_buyout < 0.9 then + average_min_buyout = (average_min_buyout + daily_min_buyout) / 2 else - item_record.average_min_buyout = (item_record.average_min_buyout * 7 + item_record.daily_min_buyout) / 8 + average_min_buyout = (average_min_buyout * 7 + daily_min_buyout) / 8 end end - - item_record.count = item_record.unit_count + item_record.daily_unit_count - item_record.count = item_record.auction_count + item_record.daily_auction_count - item_record.seen_days = item_record.seen_days + 1 - - item_record.daily_auction_count = 0 - item_record.daily_unit_count = 0 - item_record.daily_min_buyout = nil - item_record.daily_accumulated_buyout = 0 + + seen_days = seen_days + 1 + + daily_auction_count = 0 + daily_unit_count = 0 + daily_accumulated_buyout = 0 + daily_min_buyout = 0 + + item_record[2] = daily_auction_count + item_record[3] = daily_unit_count + item_record[4] = daily_accumulated_buyout + item_record[5] = daily_min_buyout + item_record[6] = seen_days + item_record[7] = EMA3 + item_record[8] = EMA7 + item_record[9] = EMA14 + item_record[10] = average_min_buyout + + private.write_record(item_key, item_record) end end - realmdata.dailypush = time() + PUSHTIME + data.next_push = time() + private.PUSH_INTERVAL end