seth
/
jam-cloud
mirror of https://bitbucket.org/jamkazam/jam-cloud.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

* VRFS-2613 - optimizing hourly cleanup by moving it to the score report routine

This commit is contained in:
Seth Call 2015-01-30 10:46:02 -06:00
parent 96bc9487c4
commit 7352fa9b19
2 changed files with 125 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
db/up/discard_scores_optimized.sql
View File

@ -1,11 +1,89 @@
DROP FUNCTION IF EXISTS discard_scores(); DROP FUNCTION IF EXISTS discard_scores();
CREATE FUNCTION discard_scores (INTEGER keep) RETURNS VOID AS $$ CREATE FUNCTION discard_scores (keep INTEGER) RETURNS VOID AS $$
BEGIN BEGIN
DELETE FROM scores WHERE score_dt > DELETE FROM scores WHERE score_dt <
(SELECT score_dt FROM scores s WHERE s.alocidispid = scores.alocidispid AND s.blocidispid = scores.blocidispid ORDER BY DESC score_dt LIMIT 1 OFFSET (keep * 2)); (SELECT score_dt FROM scores s WHERE s.alocidispid = scores.alocidispid AND s.blocidispid = scores.blocidispid ORDER BY score_dt DESC LIMIT 1 OFFSET (keep - 1));
RETURN; RETURN;
END; END;
$$ LANGUAGE plpgsql; $$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION update_current_network_scores(aloc BIGINT, bloc BIGINT) RETURNS VOID
STRICT VOLATILE AS $$
DECLARE
newscore INTEGER;
newscore_dt TIMESTAMP;
newscore_limited BOOL;
sum INTEGER;
kount INTEGER;
r RECORD;
avgscore INTEGER;
maxscore INTEGER;
minscore INTEGER;
BEGIN
-- find the 6 most recent scores
-- (supposedly newscore is the first...)
-- hybrid scheme: compute the average of some recent scores, then limit newscore to be between 4/5 and 6/5 of the average
newscore := NULL;
newscore_dt := NULL;
newscore_limited := FALSE;
sum := 0;
kount := 0;
FOR r IN SELECT score, score_dt FROM scores WHERE alocidispid = aloc AND blocidispid = bloc ORDER BY score_dt DESC LIMIT 6 LOOP
IF newscore IS NULL THEN
newscore := r.score;
newscore_dt := r.score_dt;
ELSE
sum := sum + r.score;
kount := kount + 1;
END IF;
END LOOP;
-- if no scores in query at all, then delete any current entry
IF newscore IS NULL THEN
DELETE FROM current_network_scores WHERE alocidispid = aloc AND blocidispid = bloc;
IF aloc != bloc THEN
DELETE FROM current_network_scores WHERE alocidispid = bloc AND blocidispid = aloc;
END IF;
END IF;
-- if there are scores older than newscore, then use their average to limit the range of newscore
IF kount > 0 THEN
avgscore := sum / kount;
maxscore := avgscore*6/5;
minscore := avgscore*4/5;
-- the score newscore will be inserted as the current value in current_network_scores, but we will limit it
-- to be no greater than 120% of the average and no less than 80% of the average. this will dampen wild
-- swings in the scores.
IF newscore > maxscore THEN
newscore := maxscore;
newscore_limited := TRUE;
ELSEIF newscore < minscore THEN
newscore := minscore;
newscore_limited := TRUE;
END IF;
END IF;
UPDATE current_network_scores SET score = newscore, limited = newscore_limited, score_dt = newscore_dt WHERE alocidispid = aloc AND blocidispid = bloc;
IF NOT FOUND THEN
INSERT INTO current_network_scores (alocidispid, blocidispid, score, limited, score_dt) VALUES (aloc, bloc, newscore, newscore_limited, newscore_dt);
END IF;
IF aloc != bloc THEN
UPDATE current_network_scores SET score = newscore, limited = newscore_limited, score_dt = newscore_dt WHERE alocidispid = bloc AND blocidispid = aloc;
IF NOT FOUND THEN
INSERT INTO current_network_scores (alocidispid, blocidispid, score, limited, score_dt) VALUES (bloc, aloc, newscore, newscore_limited, newscore_dt);
END IF;
END IF;
-- keep the scores table clean, meaning only up to the most 5 recent scores per group & direction (scorer)
DELETE FROM scores WHERE alocidispid = aloc AND blocidispid = bloc AND scorer = 0 AND score_dt <
(SELECT score_dt FROM scores s WHERE s.alocidispid = aloc AND s.blocidispid = bloc AND s.scorer = 0 ORDER BY score_dt DESC LIMIT 1 OFFSET 4);
DELETE FROM scores WHERE alocidispid = bloc AND blocidispid = aloc AND scorer = 1 AND score_dt <
(SELECT score_dt FROM scores s WHERE s.alocidispid = bloc AND s.blocidispid = aloc AND s.scorer = 1 ORDER BY score_dt DESC LIMIT 1 OFFSET 4);
END;
$$ LANGUAGE plpgsql;

70
ruby/spec/jam_ruby/models/score_spec.rb
View File

@ -527,24 +527,25 @@ describe Score do
end end
it "discards over 5 items" do it "discards over 5 items" do
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 6.days.ago)
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 5.days.ago)
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 4.days.ago)
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 3.days.ago)
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 2.days.ago)
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, nil)
Score.count.should == 12 Score.count.should == 10
Score.connection.execute("SELECT discard_scores(5)").check
Score.count.should == 12
Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 26, nil) Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 20, 1.days.ago)
Score.connection.execute("UPDATE scores set created_at = TIMESTAMP '#{2.days.ago}' WHERE score = 26").cmdtuples.should == 2
Score.connection.execute("SELECT discard_scores(5)").check Score.count.should == 10
Score.count.should == 12
Score.connection.execute("SELECT * FROM scores WHERE score = 20").ntuples.should == 12 # make a score older than all the rest; it should get whacked
Score.connection.execute("SELECT * FROM scores WHERE scorer = 0").ntuples.should == 6 Score.createx(LOCA, NODEA, ADDRA, LOCB, NODEB, ADDRB, 26, 7.days.ago)
Score.connection.execute("SELECT * FROM scores WHERE scorer = 1").ntuples.should == 6
Score.count.should == 10
Score.connection.execute("SELECT * FROM scores WHERE score = 20").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE scorer = 0").ntuples.should == 5
Score.connection.execute("SELECT * FROM scores WHERE scorer = 1").ntuples.should == 5
Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil) Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil)
@ -554,18 +555,35 @@ describe Score do
Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil) Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil)
Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil) Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 22, nil)
Score.count.should == 24 Score.count.should == 20
Score.connection.execute("SELECT discard_scores(5)").check
Score.count.should == 24
Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 36, nil) Score.connection.execute("SELECT * FROM scores WHERE score = 22").ntuples.should == 10
Score.connection.execute("UPDATE scores set created_at = TIMESTAMP '#{2.days.ago}' WHERE score = 36").cmdtuples.should == 2 Score.connection.execute("SELECT * FROM scores WHERE score = 20").ntuples.should == 10
Score.connection.execute("SELECT discard_scores(5)").check Score.connection.execute("SELECT * FROM scores WHERE scorer = 0").ntuples.should == 10
Score.count.should == 24 Score.connection.execute("SELECT * FROM scores WHERE scorer = 1").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE score = 22").ntuples.should == 12
Score.connection.execute("SELECT * FROM scores WHERE score = 22 AND scorer = 0").ntuples.should == 6
Score.connection.execute("SELECT * FROM scores WHERE score = 22 AND scorer = 1").ntuples.should == 6
Score.createx(LOCB, NODEB, ADDRB, LOCA, NODEA, ADDRA, 36, 7.days.ago)
Score.count.should == 20
Score.connection.execute("SELECT * FROM scores WHERE score = 22").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE score = 20").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE scorer = 0").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE scorer = 1").ntuples.should == 10
# let's create scores between a new location, and make sure they don't distrurb the data we have now
Score.createx(LOCC, NODEC, ADDRC, LOCA, NODEA, ADDRA, 10, nil)
Score.count.should == 22
Score.createx(LOCC, NODEC, ADDRC, LOCA, NODEA, ADDRA, 10, nil)
Score.createx(LOCC, NODEC, ADDRC, LOCA, NODEA, ADDRA, 10, nil)
Score.createx(LOCC, NODEC, ADDRC, LOCA, NODEA, ADDRA, 10, nil)
Score.createx(LOCC, NODEC, ADDRC, LOCA, NODEA, ADDRA, 10, nil)
Score.count.should == 30
Score.connection.execute("SELECT * FROM scores WHERE score = 20").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE score = 22").ntuples.should == 10
Score.connection.execute("SELECT * FROM scores WHERE score = 10").ntuples.should == 10
end end
end end
end end