PostgreSQL 8.4 Update

LOCAL DEVELOPER DAY '09 /Winter

( @ , 2009 )

http://www.local.gr.jp/?php%2Fhistory%2F%C2%E803%B2%F3%28LDD%2709%2FWinter%29

PostgreSQL / NTT OSS

2009.2.14

2

PostgreSQL 8.4

1977

20032004

2005

2006

2007

2008

2009

2000

19961986

PostgreSQL

6.0

PostgreSQL

6.0

7.37.3

7.47.4

8.0

8.3

8.4 2009 7

8.48.4

IngressIngress

POSTGRESPOSTGRES

•

• (PITR)

••Windows

•

• (PITR)

••Windows

8.0

••2 (2PC)

•

••2 (2PC)

•

8.1

•CPU

•

••GIN:

•CPU

•

••GIN:

8.2

•HOT:

••VACUUM /

•HOT:

••VACUUM /

8.3

3

8.4

8.4

8.3

8.5

/

XMLHOT autovacuum

2008

2009

2010 ?

WITH SQL

Window Visibility Map

pgtune

/

pg_reorg

4

PostgreSQL 8.4 : /

/

WITH

Window

/

SQL

Visibility Map

pgtune

pg_reorg

/

5

:

CHECK

8.2

8.4

constraint_exclusion = partition

CHECK

EXECUTE USING in PL/pgSQL

INSERT

4× ( )

• SELECT, UPDATE,

DELETE

•

1 2 3 4

/

6

: (EXECUTE USING)

CREATE FUNCTION insert_trigger()RETURNS TRIGGER AS

$$DECLAREpart text;

BEGINpart := 'tbl_' ||

to_char(new.insert_time, 'YYYYMM');EXECUTE 'INSERT INTO ' || part ||

' VALUES(($1).*)' USING new;RETURN NULL;

END;$$LANGUAGE plpgsql;

: tbl_[ ][ ]

: tbl

1 2 3 4

(8.3

IF )

CHECK('2009-01-01' <= insert_time

AND insert_time < '2009-02-01')

/

CHECK

CREATE TRIGGER BEFORE INSERT

( )

INSERT

EXECUTE USING

SQL new

7

:

effective_io_concurrency

I/O

RAID

I/O

Bitmap Heap Scan

◎ 1 SQL

×

posix_fadvise(WILLNEED)

POSIX ( Linux) / Windows

/

OLTP

OLAP

8

pg_restore

-m, --multi-thread = number-of-threads

PostgreSQL

CPU

/

R R R Rpg_restore

PostgreSQL

( )

pg_dump--format=custom

( )

D

9

PostgreSQL 8.4 :

/

WITH

Window

/

SQL

Visibility Map

pgtune

pg_reorg

10

WITH ( )

WITH, WITH RECURSIVE

ID=# TABLE tree;id | parent----+--------1 |2 | 13 | 14 | 3...

=# WITH RECURSIVE r AS (SELECT * FROM tree WHERE id = 1

UNION ALLSELECT tree.* FROM tree, r WHERE tree.parent = r.id

)SELECT * FROM r ORDER BY id;

id | parent----+--------1 |2 | 13 | 14 | 3

(4 rows)

self-join

1

2 3

4

11

WITH ( ) :

WITH RECURSIVE Z(IX, IY, CX, CY, X, Y, I) AS (

SELECT IX, IY, X::float, Y::float, X::float, Y::float, 0FROM (SELECT -2.2 + 0.031 * i, i

FROM generate_series(0, 101) AS i) AS xgen(x, ix),(SELECT -1.5 + 0.031 * i, i

FROM generate_series(0, 101) AS i) AS ygen(y, iy)

UNION ALL

SELECT IX, IY, CX, CY,

X * X - Y * Y + CX AS X, Y * X * 2 + CY, I + 1FROM Z

WHERE X * X + Y * Y < 16::floatAND I < 100

)

SELECT array_to_string(array_agg(substring(' .,,,-----++++%%%%@@@@#### ',least(greatest(I,1),27), 1)),'')

FROM (SELECT IX, IY, max(I) AS I

FROM Z

GROUP BY IY, IXORDER BY IY, IX

) AS ZTGROUP BY IY

ORDER BY IY;

ML (!?)

Q. ?

12

WITH ( ) : ...............................................................................................................................................................................................................................................................................................................................................................

....................................................,,,,,,,,,.................................................................................,,,,,,,,,,,,,,,,,,...........................................................................,,,,,,,,,,,,,,,,,,,,,,,,......................................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,..................................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...............................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,............................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,..........................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,........................................................,,,,,,,,,,,,,,,,,,,,,,,,--,,,,,,,,,,,,,,,,,,,,......................................................,,,,,,,,,,,,,,,,,,,,,,,,,,-+--,,,,,,,,,,,,,,,,,,,...................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----,,,,,,,,,,,,,,,,,,,.................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,--- -----,,,,,,,,,,,,,,,,,..............................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---++--++,,,,,,,,,,,,,,,,,,............................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----%++---,,,,,,,,,,,,,,,,,..........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%----,,,,,,,,,,,,,,,,,,........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----- %%+----,,,,,,,,,,,,,,,,,,.....................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---%-+% ----,,,,,,,,,,,,,,,,,,,...................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+ +## %+%---,,,,,,,,,,,,,,,,,,.................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----# # +---,,,,,,,,,,,,,,,,,,...............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------% %-----,,,,,,,,,,,,,,,,,.............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---------+ ------,,,,,,,,,,,,,,,,,...........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----------+@ +-----------,,,,,,,,,,,,.........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----@-------++ ++-----------,,,,,,,,,,,,.......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--+@% ---+ +@%%@ %%+@+@%------+-,,,,,,,,,,,......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---- # ++% % @-----++--,,,,,,,,,,,...................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----+ % %%++ %+%@-,,,,,,,,,,,..................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+# #% ++-,,,,,,,,,,,,................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,------+ @---,,,,,,,,,,,,..............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------++% ---,,,,,,,,,,,,.............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+ + %+---,,,,,,,,,,,,,...........,,,,,,,,,,,,,,,,,,,,,--------------------@ +----,,,,,,,,,,,,..........,,,,,,,,,,,,,,,,,,,,,,- +-----------------+ ----,,,,,,,,,,,,..........,,,,,,,,,,,,,,,,,,,,,--++------+---------+% +++--,,,,,,,,,,,,........,,,,,,,,,,,,,,,,,,,,,,--%+-----++--------- #+-,,,,,,,,,,,,.......,,,,,,,,,,,,,,,,,,,,,,----#%++--+@ -+-----+% --,,,,,,,,,,,,.......,,,,,,,,,,,,,,,,,,,,,,-----+## ++@ + +----% +--,,,,,,,,,,,,,......,,,,,,,,,,,,,,,,,,,,,,------+@ @ @@++++# +--,,,,,,,,,,,,,......,,,,,,,,,,,,,,,,,,,,,-------% #++% -,,,,,,,,,,,,,.....,,,,,,,,,,,,,,,,,,,,,------++%# %%@ %-,,,,,,,,,,,,,,....,,,,,,,,,,,,,,,,,,,--------+ % +--,,,,,,,,,,,,,,....,,,,,,,,,,,,,,,,,,-----+--++@ # --,,,,,,,,,,,,,,...,,,,,,,,,,,,,,,,,-------%+++% @--,,,,,,,,,,,,,,,...,,,,,,,,,,,-------------+ @#@ ---,,,,,,,,,,,,,,,...,,,,,,,,,---@--------@-+% +---,,,,,,,,,,,,,,,...,,,,,------- +-++++-+%%% +----,,,,,,,,,,,,,,,...,,,,,,------%--------++% +----,,,,,,,,,,,,,,,...,,,,,,,,,,--+----------++# ---,,,,,,,,,,,,,,,...,,,,,,,,,,,,------------+@@@% +--,,,,,,,,,,,,,,,...,,,,,,,,,,,,,,,,,------- +++% %--,,,,,,,,,,,,,,,....,,,,,,,,,,,,,,,,,,---------+@ @ --,,,,,,,,,,,,,,....,,,,,,,,,,,,,,,,,,,,------- # %@ +--,,,,,,,,,,,,,,....,,,,,,,,,,,,,,,,,,,,,-------++@ %+ %-,,,,,,,,,,,,,,.....,,,,,,,,,,,,,,,,,,,,,------- %++% %-,,,,,,,,,,,,,......,,,,,,,,,,,,,,,,,,,,,,------+# %# #@ ++++ +--,,,,,,,,,,,,,.......,,,,,,,,,,,,,,,,,,,,,,-----+ %%++% +@+----+ +--,,,,,,,,,,,,,.......,,,,,,,,,,,,,,,,,,,,,,,---%+++--+#+--------% #--,,,,,,,,,,,,........,,,,,,,,,,,,,,,,,,,,,,--++-----%%--------- @#--,,,,,,,,,,,,.........,,,,,,,,,,,,,,,,,,,,,---------------------+@ +-++,,,,,,,,,,,,...........,,,,,,,,,,,,,,,,,,,,,--------------------+ ----,,,,,,,,,,,,............,,,,,,,,,,,,,,,,,,,,----,,,------------- #+----,,,,,,,,,,,,.............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------+ + +---,,,,,,,,,,,,,..............,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+%# #---,,,,,,,,,,,,................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,------+# @ @---,,,,,,,,,,,,.................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+# + @--,,,,,,,,,,,,..................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+% %+@ %+-+ +++%-,,,,,,,,,,,.....................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----% %@++ # % -----++-,,,,,,,,,,,,......................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-- ++ ---+ + +%@ %++++++------%-,,,,,,,,,,,.........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---- -------++ +------------,,,,,,,,,,,,..........................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----------+% +--------,,,,,,,,,,,,,,,.............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,--------+# -----,,,,,,,,,,,,,,,,,,..............................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-------+ #----,,,,,,,,,,,,,,,,,,.................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,----+% %#---,,,,,,,,,,,,,,,,,,,..................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+%+%@ %+%%--,,,,,,,,,,,,,,,,,,.....................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---+-+% %----,,,,,,,,,,,,,,,,,,.......................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%@+---,,,,,,,,,,,,,,,,,,,.........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----+%----,,,,,,,,,,,,,,,,,,...........................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,-----%+ +--,,,,,,,,,,,,,,,,,..............................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,---++----,,,,,,,,,,,,,,,,,................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,---@-----,,,,,,,,,,,,,,,,,..................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,-----,,,,,,,,,,,,,,,,,,,...................................................,,,,,,,,,,,,,,,,,,,,,,,,,,--%,,,,,,,,,,,,,,,,,,,,......................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,.........................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,..........................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,............................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,..............................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,..................................................................,,,,,,,,,,,,,,,,,,,,,,,,,,,,.....................................................................,,,,,,,,,,,,,,,,,,,,,,,,..........................................................................,,,,,,,,,,,,,,,,,..................................................................................,,,,....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................

WITH SQL

A.

13

WITH (vs. CONNECT BY)

Oracle CONNECT BY / contrib/tablefunc connectby()

SELECT level, empno, mgr, ename, job

FROM emp

START WITH mgr IS NULL

CONNECT BY PRIOR empno = mgr

SELECT t.level, e.empno, e.mgr, e.ename, e.job

FROM emp e, (SELECT * FROM

connectby('emp', 'empno', 'mgr', 'mgr', 7839, 0, '~')

AS cb(empno int, mgr int, level int, branch text, pos int)) t

WHERE e.empno = t.empno;

WITH RECURSIVE r AS (

SELECT 1 AS level, * FROM emp WHERE mgr IS NULL

UNION ALL

SELECT r.level + 1, emp.* FROM emp, r WHERE r.empno = emp.mgr

)

SELECT * FROM r;

Oracle : CONNECT BY

PostgreSQL ~8.3 : connectby()

PostgreSQL 8.4 : WITH RECURSIVESQL

14

Window :

GROUP BY

(...) OVER (PARTITION BY ...)

(...) OVER (ORDER BY ...)

first_value(value)

lead(value, ...)

lag(value, ...)

(1..N )ntile(N)

cume_dist()(% )

percent_rank()

( )dense_rank()

( )rank()

row_number()

self-join

15

Window :

10%

SELECT ... FROM

(SELECT *, cume_dist() OVER (ORDER BY i) AS rank

FROM tbl) AS t

WHERE rank <= 0.1;

SELECT row_number() OVER (), *

FROM (SELECT * FROM tbl ORDER BY sortkey) AS t;

LIMIT 10%

LIMIT count(*) / 10

SELECT id, , ,

rank() OVER (PARTITION BY ORDER BY )

FROM tbl;

16

WITH + Window (1/4)

pg_locks

→

pg_locks=# SELECT locktype, database, relation, page, tuple,

transactionid, pid, mode, grantedFROM pg_locks WHERE NOT granted;

-[ RECORD 1 ]-+--------------

locktype | transactioniddatabase |

relation |

page |tuple |

transactionid | 9930pid | 3976

mode | ShareLock

granted | f

-[ RECORD 2 ]-+--------------

locktype | tupledatabase | 21509

relation | 22126

page | 0tuple | 1

transactionid |pid | 1992

mode | ExclusiveLock

granted | f

17

WITH + Window (2/4)

locktag()

CREATE FUNCTION locktag(pg_catalog.pg_locks) RETURNS text AS

$$

SELECT $1.locktype || ' ' ||CASE $1.locktype

WHEN 'relation' THEN $1.database || ' ' || $1.relationWHEN 'extend' THEN $1.database || ' ' || $1.relation

WHEN 'page' THEN $1.database || ' ' || $1.relation || ' ' || $1.page

WHEN 'tuple' THEN $1.database || ' ' || $1.relation || ' ' || $1.page|| ' ' || $1.tuple

WHEN 'transactionid' THEN $1.transactionid::textWHEN 'virtualxid' THEN $1.virtualxid

-- FIXME: for object, userlock and advisoryEND

$$

LANGUAGE sql IMMUTABLE STRICT;

18

WITH + Window (3/4)

pg_lock_chain

row_number()

CREATE VIEW pg_lock_chain AS

WITH RECURSIVE r AS (

SELECT *, locktag(pg_locks),

row_number() OVER () AS chain,

1 AS level

FROM pg_locks WHERE NOT granted

UNION ALL

SELECT s.*, locktag(s), r.chain, r.level + 1

FROM r, pg_locks s

WHERE (locktag(s) = r.locktag AND s.granted AND

NOT r.granted AND s.pid <> r.pid)

OR (s.pid = r.pid AND NOT s.granted AND r.granted)

)

SELECT * FROM r; locktag

19

=# SELECT chain, level, locktag, pid, mode, granted

FROM pg_lock_chain

ORDER BY chain, level;

chain | level | locktag | pid | mode | granted

-------+-------+-----------------------+------+---------------+---------

1 | 1 | transactionid 9930 | 3976 | ShareLock | f

1 | 2 | transactionid 9930 | 3912 | ExclusiveLock | t

2 | 1 | tuple 21509 22126 0 1 | 1992 | ExclusiveLock | f

2 | 2 | tuple 21509 22126 0 1 | 3976 | ExclusiveLock | t

2 | 3 | transactionid 9930 | 3976 | ShareLock | f

2 | 4 | transactionid 9930 | 3912 | ExclusiveLock | t

(6 rows)

WITH + Window (4/4)

pg_lock_chain

pid=3976

pid=3912

pid=1992

pid=3976

pid=3912

chain 1 chain 2

pid=3912

20

/

PostgreSQL 8.2

CREATE CAST (integer AS varchar)

WITH INOUT AS IMPLICIT;

DBMS

AS

SELECT [AS] FROM [AS]

DISTINCT GROUP BY

or

PL/pgSQL CASE

21

=# SELECT *, to_tsvector('japanese', t) FROM tbl;id | t | to_tsvector

----+------------+----------------1 | | ' ':12 | | ' ':1

(2 rows)

=# SELECT * FROM tbl WHERE to_tsvector('japanese', t) @@ to_tsquery('japanese', ' ');id | t

----+----------2 |

(1 row)

=# SELECT * FROM tbl WHERE to_tsvector('japanese', t) @@ to_tsquery('japanese', ' :*');id | t

----+------------1 | 2 |

(2 rows)

:*

N-gram ( :Ludia/Senna)

22

PostgreSQL 8.4 :

/

WITH

Window

/

SQL

Visibility Map

pgtune

pg_reorg

23

SQL :

pg_stat_statements : SQL

pg_stat_user_functions :

SQL

PostgreSQLSQL

pgFouine

( )

(HTML )

PHP

◎ ☺

×

× /

DB 30%

24

SQL :

postgresql.conf shared_preload_libraries = 'pg_stat_statements'

pgbench$ pgbench -c10 -t300 -M prepared

query mode: prepared

number of transactions actually processed: 3000/3000

SQL=# SELECT query, calls, total_time, rows FROM pg_stat_statements ORDER BY total_time DESC LIMIT 3;

query | calls | total_time | rows

--------------------------------------------------------------+-------+-------------------+------

UPDATE branches SET bbalance = bbalance + $1 WHERE bid = $2; | 3000 | 35.9654100452473 | 3000

UPDATE tellers SET tbalance = tbalance + $1 WHERE tid = $2; | 3000 | 34.7969816235719 | 3000

UPDATE accounts SET abalance = abalance + $1 WHERE aid = $2; | 3000 | 0.660384690059746 | 3000

(3 rows)

SQL

prepared

(Prepared Statement)

◎

◎ 3%

× HTML

25

Visibility Map : VACUUM

PostgreSQL VACUUM ?

8.4 VACUUM

Visibility Map

VACUUM +

Free Space Map

8.3 HOT ?

HOT =

Visibility Map =

B

C

B

A

8002

7002

20003

10001

ID

UPDATEB

C

A

7002

20003

10001

ID

VACUUM

26

VACUUM

Visibility Map (VACUUM )

8.4 VACUUMVACUUM

Free Space Map (VACUUM )

8.4DB

(< ID>[.N])

Visibility Map(< ID>_vm)

Free Space Map(< ID>_fsm)

50%0%

25%

:

1bit/1byte/

(shared_buffers) I/O

VACUUMVisibility Map

Free Space Map

27

SQL

SQL

B2

A1

valueid

ERROR: deadlock detected

DETAIL:Process 1956 waits for ShareLock on transaction 8214; blocked by process 2392.

Process 2392 waits for ShareLock on transaction 8215; blocked by process 1956.

Process 1956: UPDATE tbl SET value = 'AAA' WHERE id = 1;

Process 2392: UPDATE tbl SET value = 'BBB' WHERE id = 2;

UPDATE tbl SET value = 'AAA'

WHERE id = 1;

UPDATE tbl SET value = 'BBB'

WHERE id = 2;

UPDATE tbl SET value = 'BB'

WHERE id = 2;

UPDATE tbl SET value = 'AA'

WHERE id = 1;

BEGIN;BEGIN;

2 (pid=1956)1 (2392)

28

PostgreSQL 8.4 :

/

WITH

Window

/

SQL

Visibility Mappgtune

pg_reorg

29

pgtune :

(DW, OLTP, Web, Mixed, Desktop)

( )

default_statistics_target

constraint_exclusion

effective_cache_size

checkpoint_completion_target

checkpoint_segments

maintenance_work_mem

work_mem

wal_buffers

shared_buffers

max_connections

WAL

onpartition

163

8MB64kB

50100

12MB1MB

480MB32MB

1408MB128MB

0.90.5

120MB16MB

80100

pgtune

=Mixed, =3GB postgresql.confpostgresql.conf

postgresql.confpostgresql.conf

pgtune

http://pgfoundry.org/projects/pgtune/

in

out

30

pg_reorg :

INSERT

UPDATE

DELETE ( )

0 100 200 300

( 2GB)

pg_reorg

clusterdb

17■

274■

http://reorg.projects.postgresql.org/index-ja.html

/DB

VACUUM

PostgreSQL: 8.2, 8.3, 8.4

OS: Linux, Windows

31

pgbench :

pgbench

TPC-B

SQL

pgbench

(-T)

(-t)

(-M)

simple, extended, prepared

32

PostgreSQL 8.4 ( ) 2009 7

/

, ,

WITH , Window , /

SQL , Visibility Map,

pgtune, pg_reorg

33

TRUNCATE TRIGGER

SQL/MED

pg_terminate_backend()

contrib/auto_explain

autovacuum TOAST VACUUM

pgAdmin III pgScript

34

ToDo ( )

http://wiki.postgresql.org/wiki/Todo

PostgreSQL 8.4 Development Plan ( )

http://wiki.postgresql.org/wiki/PostgreSQL_8.4_Dev

elopment_Plan

Waiting for 8.4 ( blog)

http://www.depesz.com/