00001
package com.quadcap.sql.file;
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038
00039
00040
00041
import java.io.ByteArrayOutputStream;
00042
import java.io.File;
00043
import java.io.FileInputStream;
00044
import java.io.IOException;
00045
import java.io.InputStream;
00046
import java.io.OutputStream;
00047
import java.io.PrintWriter;
00048
00049
import java.util.Properties;
00050
00051
import com.quadcap.util.ConfigNumber;
00052
import com.quadcap.util.Debug;
00053
import com.quadcap.util.Util;
00054
00055
import com.quadcap.util.collections.LongMap;
00056
00057
import com.quadcap.crypto.KeyFactory;
00058
import com.quadcap.crypto.SymmetricKey;
00059
00060
import com.quadcap.io.MutableByteArrayInputStream;
00061
00062
import com.quadcap.sql.io.ObjectInputStream;
00063
import com.quadcap.sql.io.ObjectOutputStream;
00064
00065
import com.quadcap.sql.Version;
00066
00067
00068
00069
00070
00071
00072
00073
00074
00075 public class BlockFile implements PageManager,
SegmentManager {
00076
00077 BlockStore store;
00078
00079
00080 BlockCache cache;
00081
00082
00083 LongMap
memoryObjects = null;
00084 long memObjectCount = 100000;
00085 boolean inMemory;
00086
00087 static final int REF_SIZE = 8;
00088
00089 static final int MIN_SUBPAGE = 64;
00090
00091 static final int MAX_PAGESHIFT = 16;
00092
00093 int maxPageShift = 0;
00094
00095
00096
00097
00098
00099
00100
00101 static final int oBLOCKSIZE = 4;
00102
00103
00104 static final int oFREELIST = 8;
00105
00106
00107 static final int oLASTBLOCK = 16;
00108
00109
00110 static final int oSTREAM_START = 24;
00111 static final int MAX_STREAM = 4;
00112
00113
00114
static final int oHASH_PASSWD_start
00115 =
oSTREAM_START + (
REF_SIZE *
MAX_STREAM);
00116 static final int oHASH_PASSWD_len = 20;
00117
00118
static final int oSUBPAGE_ROOT
00119 =
oHASH_PASSWD_start +
oHASH_PASSWD_len;
00120 static final int oHEADER_SIZE = (
oSUBPAGE_ROOT +
00121 (
MAX_PAGESHIFT *
REF_SIZE));
00122
00123 int blockSize;
00124
00125
00126 long lastBlock = 0;
00127
00128 boolean readOnly;
00129
00130 PageManager[]
pageManagers =
new PageManager[
MAX_PAGESHIFT];
00131 Object
fileLock =
new Object();
00132
00133 MutableByteArrayInputStream
mbis =
new MutableByteArrayInputStream();
00134 ObjectInputStream
ois =
new ObjectInputStream(mbis);
00135 ByteArrayOutputStream
bos =
new ByteArrayOutputStream();
00136 ObjectOutputStream
oos;
00137
00138
00139
00140
00141
00142
00143
00144
00145
00146
00147 public BlockFile(String filename, String mode,
00148 Properties props,
00149
int blocksize,
00150
int cacheSize)
00151
throws IOException
00152 {
00153
00154
if (
Trace.bit(4)) {
00155
Debug.println(
"BlockFile(" + filename +
", " + mode +
00156
", blockSize = " + blocksize +
00157
", cacheSize = " + cacheSize +
")");
00158 }
00159
00160
this.inMemory =
00161 props.getProperty(
"memoryOnly",
"").equalsIgnoreCase(
"true");
00162
pageManagers[0] =
this;
00163
int psize = blocksize;
00164
maxPageShift = 0;
00165
while (psize >
MIN_SUBPAGE) {
00166
maxPageShift++;
00167 psize >>= 1;
00168 }
00169
this.readOnly = mode.equalsIgnoreCase(
"r");
00170
this.store =
makeStore(props, filename, blocksize, mode);
00171
this.blockSize =
store.
blockSize();
00172
this.cache =
new BlockCache();
00173
this.oos =
new ObjectOutputStream(
bos);
00174
if (
inMemory) {
00175
this.memoryObjects =
new LongMap(8888);
00176 }
00177
cache.
setLock(
fileLock);
00178
cache.
setReadOnly(
readOnly);
00179
cache.
init(
store, cacheSize);
00180
Block root =
getBlock(0);
00181
try {
00182
this.lastBlock = root.
readLong(
oLASTBLOCK);
00183 } finally {
00184 root.
decrRefCount();
00185 }
00186 }
00187
00188
00189 public String
signature() throws IOException {
00190 StringBuffer sb =
new StringBuffer(
"{");
00191
for (
long blk = 0; blk <=
lastBlock; blk++) {
00192
if (blk > 0) sb.append(
",");
00193
Block b =
getBlock(blk);
00194
try {
00195 sb.append(b.
signature());
00196 } finally {
00197 b.
decrRefCount();
00198 }
00199 }
00200 sb.append(
"}");
00201
return sb.toString();
00202 }
00203
00204
00205 protected BlockStore makeStore(Properties props, String filename,
00206
int blockSize, String mode)
00207
throws IOException
00208 {
00209
BlockStore s;
00210 File f = null;
00211
boolean enc =
false;
00212
boolean nullStore =
00213 props.getProperty(
"cacheOnly",
"").equalsIgnoreCase(
"true");
00214
if (
inMemory || nullStore) {
00215
if (
inMemory) {
00216 s =
new MemoryBlockStore();
00217 }
else {
00218 s =
new NullStore();
00219 }
00220 }
else {
00221 filename =
new File(filename).getAbsolutePath();
00222 f =
new File(filename);
00223
boolean exists = f.exists();
00224
if (!exists &&
readOnly) {
00225
throw new IOException(
"Not found: " + filename);
00226 }
00227 String u = props.getProperty(
"user",
00228 props.getProperty(
"jdbc.user",
""));
00229 String p = props.getProperty(
"passwd",
00230 props.getProperty(
"jdbc.passwd",
""));
00231
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241
00242
00243
00244
00245
00246
00247
boolean encrypt =
00248 props.getProperty(
"encrypt",
"").equalsIgnoreCase(
"true");
00249 enc = (exists &&
isEncrypted(f)) || (!exists && encrypt);
00250
if (enc) {
00251 s =
new EncryptedBlockStore();
00252 }
else {
00253 s =
new BlockStore();
00254 }
00255 }
00256 s.
init(f, mode,
blockSize,
fileLock);
00257
if (enc) {
00258 String u = props.getProperty(
"user",
00259 props.getProperty(
"jdbc.user",
""));
00260 String p = props.getProperty(
"passwd",
00261 props.getProperty(
"jdbc.passwd",
""));
00262
SymmetricKey key =
KeyFactory.createSymmetricKey(u +
"/" + p);
00263 s.
setKey(key);
00264 }
00265
return s;
00266 }
00267
00268 private boolean isEncrypted(File f)
throws IOException {
00269 FileInputStream fi =
new FileInputStream(f);
00270
try {
00271
return fi.read() == 0x0e;
00272 } finally {
00273 fi.close();
00274 }
00275 }
00276
00277 public long getUserBlock(
int stream)
throws IOException {
00278
synchronized (
fileLock) {
00279
if (stream >
MAX_STREAM) {
00280
throw new IOException(
"Invalid stream: " + stream);
00281 }
00282
Block root =
getBlock(0);
00283
long blockNum;
00284
int offset =
oSTREAM_START + (stream *
REF_SIZE);
00285
try {
00286 blockNum = root.
readLong(offset);
00287
if (blockNum == 0) {
00288 blockNum =
newPage();
00289 root.
writeLong(offset, blockNum);
00290 }
00291 } finally {
00292 root.
decrRefCount();
00293 }
00294
return blockNum;
00295 }
00296 }
00297
00298
00299
00300
00301 public void close() throws IOException {
00302
synchronized (
fileLock) {
00303
flush(
false);
00304
store.
close();
00305 }
00306 }
00307
00308 public final void flush(
boolean fastSync)
throws IOException {
00309
synchronized (
fileLock) {
00310
00311
if (
Trace.bit(24)) {
00312
Debug.println(
toString() +
".flush(" + fastSync +
")");
00313 }
00314
00315
if (!
readOnly) {
00316
cache.
flush();
00317
if (!fastSync) {
00318
store.
flush();
00319 }
00320 }
00321 }
00322 }
00323
00324 public String
getName() {
00325
return store.
getName();
00326 }
00327
00328 public void setLog(
Log log) {
00329
store.
setLog(log);
00330 }
00331
00332 Log getLog() {
00333
return store.
getLog();
00334 }
00335
00336
00337
00338
00339
00340
00341
00342
00343
00344 public Page getPage(
long i)
throws IOException {
00345
synchronized (
fileLock) {
00346
if (i >
lastBlock) {
00347
00348
Debug.println(
toString() +
".getPage(" + i +
"): Bad Block " +
00349
" vs lastBlock = " +
lastBlock);
00350
00351
throw new IOException(
"BlockFile(" +
store +
00352
"), Bad block: " + i +
" (" +
00353
SubPageManager.
toString(i) +
")");
00354 }
00355
return (
Page)
cache.
getCacheable(i);
00356 }
00357 }
00358
00359 public Block getBlock(
long i)
throws IOException {
00360
return (
Block)getPage(i);
00361 }
00362
00363 public void restoreBlock(
long b, byte[] buf,
int off)
throws IOException {
00364
ensureLastBlock(b);
00365
store.
restore(b, buf, off);
00366 }
00367
00368 final void ensureLastBlock(
long i)
throws IOException {
00369
if (i >
lastBlock) {
00370
lastBlock = i;
00371
Block root = getBlock(0);
00372
try {
00373 root.
writeLong(
oLASTBLOCK, i);
00374 } finally {
00375 root.
decrRefCount();
00376 }
00377 }
00378 }
00379
00380
00381
00382
00383
00384 public final int getPageSize() {
00385
return this.blockSize;
00386 }
00387
00388
00389
00390
00391 public RandomAccess getStream(
long blockRef)
throws IOException {
00392
PageManager p =
getPageManagerForPage(blockRef);
00393
return new BlockAccess(p, blockRef);
00394 }
00395
00396
00397
00398
00399
00400 public long newPage() throws IOException {
00401
synchronized (
fileLock) {
00402
Block root = getBlock(0);
00403
long ref;
00404
try {
00405 ref = root.
readLong(
oFREELIST);
00406
if (ref < 0 || ref >
lastBlock) {
00407
throw new IOException(
this +
"newPage(), bad ref: " +
00408 ref +
", lastBlock = " +
00409
lastBlock);
00410 }
00411
if (ref != 0) {
00412
Block newPage = getBlock(ref);
00413
try {
00414 root.
writeLong(
oFREELIST, newPage.
readLong(0));
00415
00416 newPage.
clear();
00417 } finally {
00418 newPage.
decrRefCount();
00419 }
00420 }
else {
00421 ref = root.
readLong(
oLASTBLOCK);
00422
Block newPage = getBlock(ref);
00423
try {
00424 newPage.
clear();
00425 } finally {
00426 newPage.
decrRefCount();
00427 }
00428
lastBlock = ref + 1;
00429 root.
writeLong(
oLASTBLOCK,
lastBlock);
00430 }
00431 } finally {
00432 root.
decrRefCount();
00433 }
00434
00435
if (
Trace.bit(5)) {
00436
Debug.println(
toString() +
".newPage() = " + ref +
"");
00437 }
00438
00439
return ref;
00440 }
00441 }
00442
00443
00444
00445
00446
00447
00448 public void freePage(
long ref)
throws IOException {
00449
PageManager p =
getPageManagerForPage(ref);
00450
if (p !=
this) {
00451 p.
freePage(ref);
00452 }
else synchronized (
fileLock) {
00453
00454
if (
Trace.bit(5)) {
00455
Debug.println(
toString() +
".freePage(" + ref +
")");
00456 }
00457
00458
if (ref <= 0 || ref >
lastBlock) {
00459
throw new IOException(
"bad ref");
00460 }
00461
Block blk = getBlock(ref);
00462
try {
00463 blk.
getDataAndReset();
00464
00465
Block root = getBlock(0);
00466
try {
00467
long freeList = root.
readLong(
oFREELIST);
00468 blk.
writeLong(0, freeList);
00469 root.
writeLong(
oFREELIST, ref);
00470 } finally {
00471 root.
decrRefCount();
00472 }
00473 } finally {
00474 blk.
decrRefCount();
00475 }
00476 }
00477 }
00478
00479 static int getPageOffset(
PageManager pm,
long ref) {
00480
int psize = pm.
getPageSize();
00481
int pnum = (
int)(
SubPageManager.fExt(ref,
SubPageManager.PAGE_NUM));
00482
return pnum * psize;
00483 }
00484
00485 static long getPageBlock(
long ref) {
00486
return SubPageManager.pageBlock(ref);
00487 }
00488
00489 static final boolean quick =
true;
00490
00491
00492
00493
00494
00495
00496
00497
00498 public long putBytes(byte[] buf)
throws IOException {
00499
PageManager pm =
getPageManagerForLen(buf.length);
00500
long seg = pm.
newPage();
00501
if (
quick && (buf.length + 8) < pm.
getPageSize()) {
00502
int off = getPageOffset(pm, seg);
00503
long blk = getPageBlock(seg);
00504
synchronized (
fileLock) {
00505
Block b = getBlock(blk);
00506
try {
00507 b.
writeLong(off, buf.length);
00508 b.
write(off+8, buf, 0, buf.length);
00509 } finally {
00510 b.
decrRefCount();
00511 }
00512 }
00513 }
else {
00514
synchronized (
fileLock) {
00515
ba.
init(pm, seg);
00516
ba.
resize(buf.length);
00517
ba.
write(0, buf, 0, buf.length);
00518 }
00519 }
00520
00521
00522
00523
return seg;
00524 }
00525
00526 BlockAccess ba =
new BlockAccess();
00527
00528
00529
00530
00531
00532
00533
00534 public byte[]
getBytes(
long seg)
throws IOException {
00535
PageManager pm =
getPageManagerForPage(seg);
00536 byte[] buf = null;
00537
int off = getPageOffset(pm, seg);
00538
long blk = getPageBlock(seg);
00539
synchronized (
fileLock) {
00540
if (
quick) {
00541
Block b = getBlock(blk);
00542
try {
00543
int len = (
int)b.
readLong(off);
00544
if (len + 8 < pm.
getPageSize()) {
00545 buf =
new byte[len];
00546 b.
read(off+8, buf, 0, len);
00547 }
00548 } finally {
00549 b.
decrRefCount();
00550 }
00551 }
00552
if (buf == null) {
00553
ba.
init(pm, seg);
00554 buf =
new byte[(
int)(
ba.
size())];
00555
ba.
read(0, buf, 0, buf.length);
00556 }
00557 }
00558
00559
00560
00561
return buf;
00562 }
00563
00564
00565
00566
00567
00568
00569
00570
00571 public void updateBytes(
long seg, byte[] buf)
throws IOException {
00572
00573
00574
00575
PageManager pm =
getPageManagerForPage(seg);
00576
synchronized (
fileLock) {
00577
if (
quick && (buf.length + 8) < pm.
getPageSize()) {
00578
Block b = getBlock(getPageBlock(seg));
00579
try {
00580
int off = getPageOffset(pm, seg);
00581
long oldLen = b.
readLong(off);
00582
if ((oldLen + 8) < pm.
getPageSize()) {
00583 b.
writeLong(off, buf.length);
00584 b.
write(off+8, buf, 0, buf.length);
00585
return;
00586 }
00587 } finally {
00588 b.
decrRefCount();
00589 }
00590 }
00591
00592
ba.
init(pm, seg);
00593
if (
ba.
size() != buf.length) {
00594
ba.
resize(buf.length);
00595 }
00596
ba.
write(0, buf, 0, buf.length);
00597 }
00598 }
00599
00600
00601
00602
00603
00604
00605
00606
00607
00608
00609 public void freeStream(
long page)
throws IOException {
00610
synchronized (
fileLock) {
00611 getStream(page).
resize(0);
00612 freePage(page);
00613 }
00614 }
00615 public void freeSegment(
long seg)
throws IOException {
00616 freeStream(seg);
00617 }
00618
00619
00620
00621
00622
00623
00624
00625
00626
00627
00628 public RandomAccessInputStream getInputStream(
long block)
00629
throws IOException
00630 {
00631
return new RandomAccessInputStream(getStream(block));
00632 }
00633
00634
00635
00636
00637
00638
00639
00640
00641
00642
00643 public RandomAccessOutputStream getOutputStream(
long block)
00644
throws IOException
00645 {
00646
return new RandomAccessOutputStream(getStream(block));
00647 }
00648
00649
00650
00651
00652 public PageManager getPageManager(
int i)
throws IOException {
00653
if (i < 0 || i >
maxPageShift) {
00654
throw new IOException(
"Bad page shift: " + i);
00655 }
00656
PageManager pm =
pageManagers[i];
00657
if (pm == null) {
00658 pageManagers[i] = pm =
new SubPageManager(
this, i, 0);
00659 }
00660
return pm;
00661 }
00662
00663 public PageManager getPageManagerForPage(
long page)
throws IOException {
00664
return getPageManager(
SubPageManager.pageShift(page));
00665 }
00666
00667 public PageManager getPageManagerForLen(
long len)
throws IOException {
00668
int pageShift = 0;
00669
int siz =
blockSize >> 1;
00670
while (len < siz && pageShift <
maxPageShift-1) {
00671 siz >>= 1;
00672 pageShift++;
00673 }
00674
if (pageShift > 0) {
00675 pageShift--;
00676 }
00677
return getPageManager(pageShift);
00678 }
00679
00680 public void revert() throws IOException {
00681
cache.
revert();
00682
Block root = getBlock(0);
00683
try {
00684
this.lastBlock = root.
readLong(
oLASTBLOCK);
00685 } finally {
00686 root.
decrRefCount();
00687 }
00688
store.
setLength(
this.
lastBlock *
blockSize);
00689 }
00690
00691 public void showCache(PrintWriter os) {
00692
cache.
show(os);
00693 }
00694
00695 public String
toString() {
00696
return "BlockFile(" +
store +
")";
00697 }
00698
00699 public void clearModified() throws IOException {
00700
store.
clearModified();
00701 }
00702
00703 public long getSize() {
00704
return lastBlock *
blockSize;
00705 }
00706
00707 public Object
getLock() {
00708
return fileLock;
00709 }
00710
00711 public int getBlockSize() {
00712
return blockSize;
00713 }
00714
00715
00716
00717
00718
00719
00720
00721
00722
00723 public Object
getObject(
long ref)
throws IOException {
00724 Object ret = null;
00725
synchronized (
fileLock) {
00726
if (
inMemory) {
00727 ret =
memoryObjects.get(ref);
00728 }
else {
00729
try {
00730 byte[] buf = getBytes(ref);
00731
mbis.reset(buf);
00732 ret =
ois.readObject();
00733 }
catch (ClassNotFoundException e) {
00734
throw new DatafileException(e);
00735 }
00736 }
00737 }
00738
00739
return ret;
00740 }
00741
00742
00743
00744
00745
00746
00747
00748
00749 public long putObject(Object obj)
throws IOException {
00750
synchronized (
fileLock) {
00751
if (
inMemory) {
00752
memoryObjects.put(++
memObjectCount, obj);
00753
00754
return memObjectCount;
00755 }
00756
bos.reset();
00757
oos.writeObject(obj);
00758
oos.flush();
00759 byte[] buf =
bos.toByteArray();
00760
long ret = putBytes(buf);
00761
return ret;
00762 }
00763 }
00764
00765
00766
00767
00768
00769
00770
00771
00772 public void updateObject(
long seg, Object obj)
throws IOException {
00773
00774
synchronized (
fileLock) {
00775
if (
inMemory) {
00776
memoryObjects.put(seg, obj);
00777 }
else {
00778
bos.reset();
00779
oos.writeObject(obj);
00780
oos.flush();
00781 byte[] buf =
bos.toByteArray();
00782 updateBytes(seg, buf);
00783 }
00784 }
00785 }
00786
00787
00788
00789
00790
00791
00792
00793 public void removeObject(
long ref)
throws IOException {
00794
00795
synchronized (
fileLock) {
00796
if (
inMemory) {
00797
memoryObjects.remove(ref);
00798 }
else {
00799 freeSegment(ref);
00800 }
00801 }
00802 }
00803
00804
00805
00806
00807 public void dump() throws IOException {
00808
Block root = getBlock(0);
00809
try {
00810
Debug.println(
"Magic: " +
00811 Integer.toHexString(root.
readShort(0)));
00812
Debug.println(
"Major version: " + root.
readByte(2));
00813
Debug.println(
"Minor version: " + root.
readByte(3));
00814
Debug.println(
"Block Size: " + root.
readInt(
oBLOCKSIZE));
00815
Debug.println(
"FreeList: " + root.
readLong(
oFREELIST));
00816 java.util.BitSet free =
new java.util.BitSet();
00817
for (
long ref = root.
readLong(
oFREELIST); ref != 0; ) {
00818 free.set((
int)ref);
00819
Block newPage = getBlock(ref);
00820
try {
00821 ref = newPage.
readLong(0);
00822 } finally {
00823 newPage.
decrRefCount();
00824 }
00825 }
00826
int first = -2;
00827
int last = -2;
00828
for (
int i = free.nextSetBit(0); i >= 0; i = free.nextSetBit(i+1)) {
00829
if (i != last+1) {
00830
if (first >= 0) {
00831
if (first == last) {
00832
Debug.println(
"Free: " + first);
00833 }
else {
00834
Debug.println(
"Free: " + first +
" - " + last);
00835 }
00836 }
00837 first = i;
00838 last = i;
00839 }
00840 }
00841
if (first >= 0) {
00842
if (first == last) {
00843
Debug.println(
"Free: " + first);
00844 }
else {
00845
Debug.println(
"Free: " + first +
" - " + last);
00846 }
00847 }
00848
long lastBlock_ = root.
readLong(
oLASTBLOCK);
00849
Debug.println(
"Last Block: " + root.
readLong(
oLASTBLOCK));
00850
long waste = free.cardinality() *
blockSize;
00851
long total = lastBlock_ *
blockSize;
00852
double pct = 100 * ((
double)waste) / ((
double)total);
00853 String pcts = java.text.NumberFormat.getInstance().format(pct);
00854
Debug.println(
"Fragmentation: " + free.cardinality() +
" of " +
00855 lastBlock_ +
" blocks = " + pcts +
"% waste");
00856
Debug.println(
"Last Block: " + root.
readLong(
oLASTBLOCK));
00857
for (
int i = 0; i <
MAX_STREAM; i++) {
00858
Debug.println(
"Stream " + i +
": " +
00859 root.
readLong(
oSTREAM_START + i*
REF_SIZE));
00860 }
00861 byte[] hp =
new byte[
oHASH_PASSWD_len];
00862 root.
read(
oHASH_PASSWD_start, hp, 0,
oHASH_PASSWD_len);
00863
Debug.println(
"Hash Passwd: " +
Util.hexBytes(hp));
00864
for (
int i = 0; i < 16; i++) {
00865
Debug.println(
"PM " + i +
" root: " +
00866
SubPageManager.toString(root.
readLong(
oSUBPAGE_ROOT + i *
REF_SIZE)));
00867 }
00868 } finally {
00869 root.
decrRefCount();
00870 }
00871 }
00872 public static void main(String[] args) {
00873
try {
00874
BlockFile f =
new BlockFile(args[0],
"r",
00875
new Properties(),
00876 8192, 256);
00877 f.
dump();
00878 }
catch (Throwable t) {
00879
Debug.print(t);
00880 }
00881 }
00882
00883
00884 }