Commit 8c573ff6 authored by OzzieIsaacs's avatar OzzieIsaacs

Unrar/tar and unzip on server side

parent 6e714f3f
This diff is collapsed.
This diff is collapsed.
......@@ -17,6 +17,8 @@
*/
/* global bitjs */
var start=0;
if (window.opera) {
window.console.log = function(str) {
opera.postError(str);
......@@ -127,11 +129,11 @@ var createURLFromArray = function(array, mimeType) {
// This would save 25% on memory since base64-encoded strings are 4/3 the size of the binary
kthoom.ImageFile = function(file) {
this.filename = file.filename;
var fileExtension = file.filename.split(".").pop().toLowerCase();
/*var fileExtension = file.filename.split(".").pop().toLowerCase();
var mimeType = fileExtension === "png" ? "image/png" :
(fileExtension === "jpg" || fileExtension === "jpeg") ? "image/jpeg" :
fileExtension === "gif" ? "image/gif" : null;
this.dataURI = createURLFromArray(file.fileData, mimeType);
fileExtension === "gif" ? "image/gif" : null;*/
this.dataURI = file.fileData; // createURLFromArray(file.fileData, mimeType);
this.data = file;
};
......@@ -270,50 +272,22 @@ kthoom.setProgressMeter = function(pct, optLabel) {
}
function loadFromArrayBuffer(ab) {
var start = (new Date).getTime();
var h = new Uint8Array(ab, 0, 10);
var pathToBitJS = "../../static/js/";
if (h[0] === 0x52 && h[1] === 0x61 && h[2] === 0x72 && h[3] === 0x21) { //Rar!
unarchiver = new bitjs.archive.Unrarrer(ab, pathToBitJS);
} else if (h[0] === 80 && h[1] === 75) { //PK (Zip)
unarchiver = new bitjs.archive.Unzipper(ab, pathToBitJS);
} else { // Try with tar
unarchiver = new bitjs.archive.Untarrer(ab, pathToBitJS);
var f=[];
f.fileData=ab.content;
f.filename=ab.name;
// add any new pages based on the filename
if (imageFilenames.indexOf(f.filename) === -1) {
imageFilenames.push(f.filename);
imageFiles.push(new kthoom.ImageFile(f));
}
// Listen for UnarchiveEvents.
if (unarchiver) {
unarchiver.addEventListener(bitjs.archive.UnarchiveEvent.Type.PROGRESS,
function(e) {
var percentage = e.currentBytesUnarchived / e.totalUncompressedBytesInArchive;
totalImages = e.totalFilesInArchive;
kthoom.setProgressMeter(percentage, "Unzipping");
// display nav
lastCompletion = percentage * 100;
});
unarchiver.addEventListener(bitjs.archive.UnarchiveEvent.Type.EXTRACT,
function(e) {
// convert DecompressedFile into a bunch of ImageFiles
if (e.unarchivedFile) {
var f = e.unarchivedFile;
// add any new pages based on the filename
if (imageFilenames.indexOf(f.filename) === -1) {
imageFilenames.push(f.filename);
imageFiles.push(new kthoom.ImageFile(f));
}
}
// display first page if we haven't yet
if (imageFiles.length === currentImage + 1) {
updatePage();
}
});
unarchiver.addEventListener(bitjs.archive.UnarchiveEvent.Type.FINISH,
function() {
var diff = ((new Date).getTime() - start) / 1000;
console.log("Unarchiving done in " + diff + "s");
});
unarchiver.start();
} else {
alert("Some error");
var percentage = (ab.page+1) / (ab.last+1);
totalImages = ab.last+1;
kthoom.setProgressMeter(percentage, "Unzipping");
lastCompletion = percentage * 100;
// display first page if we haven't yet
if (imageFiles.length === currentImage + 1) {
updatePage();
}
}
......@@ -524,69 +498,90 @@ function keyHandler(evt) {
}
}
function init(filename) {
if (!window.FileReader) {
alert("Sorry, kthoom will not work with your browser because it does not support the File API. Please try kthoom with Chrome 12+ or Firefox 7+");
} else {
var request = new XMLHttpRequest();
request.open("GET", filename);
request.responseType = "arraybuffer";
request.setRequestHeader("X-Test", "test1");
request.setRequestHeader("X-Test", "test2");
request.addEventListener("load", function(event) {
if (request.status >= 200 && request.status < 300) {
loadFromArrayBuffer(request.response);
} else {
console.warn(request.statusText, request.responseText);
}
});
request.send();
kthoom.initProgressMeter();
document.body.className += /AppleWebKit/.test(navigator.userAgent) ? " webkit" : "";
updateScale(true);
kthoom.loadSettings();
$(document).keydown(keyHandler);
$(window).resize(function() {
var f = (screen.width - innerWidth < 4 && screen.height - innerHeight < 4);
getElem("titlebar").className = f ? "main" : "";
updateScale();
});
$("#mainImage").click(function(evt) {
// Firefox does not support offsetX/Y so we have to manually calculate
// where the user clicked in the image.
var mainContentWidth = $("#mainContent").width();
var mainContentHeight = $("#mainContent").height();
var comicWidth = evt.target.clientWidth;
var comicHeight = evt.target.clientHeight;
var offsetX = (mainContentWidth - comicWidth) / 2;
var offsetY = (mainContentHeight - comicHeight) / 2;
var clickX = !!evt.offsetX ? evt.offsetX : (evt.clientX - offsetX);
var clickY = !!evt.offsetY ? evt.offsetY : (evt.clientY - offsetY);
// Determine if the user clicked/tapped the left side or the
// right side of the page.
var clickedPrev = false;
switch (kthoom.rotateTimes) {
case 0:
clickedPrev = clickX < (comicWidth / 2);
break;
case 1:
clickedPrev = clickY < (comicHeight / 2);
break;
case 2:
clickedPrev = clickX > (comicWidth / 2);
break;
case 3:
clickedPrev = clickY > (comicHeight / 2);
break;
}
if (clickedPrev) {
showPrevPage();
} else {
showNextPage();
}
});
function ImageLoadCallback(event) {
var jso=this.response;
if (jso.page !== jso.last)
{
// var secRequest = new XMLHttpRequest();
this.open("GET", this.fileid + "/"+(jso.page+1));
this.addEventListener("load",ImageLoadCallback);
this.send();
}
else
{
var diff = ((new Date).getTime() - start)/1000;
console.log('Transfer done in ' + diff + 's');
}
loadFromArrayBuffer(jso);
}
function init(fileid) {
start = (new Date).getTime();
var request = new XMLHttpRequest();
request.open("GET", fileid);
request.responseType = "json";
request.fileid=fileid.substring(0,fileid.length - 2);
request.addEventListener("load",ImageLoadCallback);/* function(event) {
var jso=request.response;
if (jso.page!=jso.length)
{
// var secRequest = new XMLHttpRequest();
request.open("GET", fileid + "/../"+(jso.page+1));
request.send();
//secRequest.responseType = "json";
//finished;
}
loadFromArrayBuffer(jso);
// var byteArray = new Uint8Array(request.response);
// if you want to access the bytes:
});*/
request.send();
kthoom.initProgressMeter();
document.body.className += /AppleWebKit/.test(navigator.userAgent) ? " webkit" : "";
updateScale(true);
kthoom.loadSettings();
$(document).keydown(keyHandler);
$(window).resize(function() {
var f = (screen.width - innerWidth < 4 && screen.height - innerHeight < 4);
getElem("titlebar").className = f ? "main" : "";
updateScale();
});
$("#mainImage").click(function(evt) {
// Firefox does not support offsetX/Y so we have to manually calculate
// where the user clicked in the image.
var mainContentWidth = $("#mainContent").width();
var mainContentHeight = $("#mainContent").height();
var comicWidth = evt.target.clientWidth;
var comicHeight = evt.target.clientHeight;
var offsetX = (mainContentWidth - comicWidth) / 2;
var offsetY = (mainContentHeight - comicHeight) / 2;
var clickX = !!evt.offsetX ? evt.offsetX : (evt.clientX - offsetX);
var clickY = !!evt.offsetY ? evt.offsetY : (evt.clientY - offsetY);
// Determine if the user clicked/tapped the left side or the
// right side of the page.
var clickedPrev = false;
switch (kthoom.rotateTimes) {
case 0:
clickedPrev = clickX < (comicWidth / 2);
break;
case 1:
clickedPrev = clickY < (comicHeight / 2);
break;
case 2:
clickedPrev = clickX > (comicWidth / 2);
break;
case 3:
clickedPrev = clickY > (comicHeight / 2);
break;
}
if (clickedPrev) {
showPrevPage();
} else {
showNextPage();
}
});
}
This diff is collapsed.
/**
* untar.js
*
* Copyright(c) 2011 Google Inc.
*
* Reference Documentation:
*
* TAR format: http://www.gnu.org/software/automake/manual/tar/Standard.html
*/
// This file expects to be invoked as a Worker (see onmessage below).
importScripts('io.js');
importScripts('archive.js');
// Progress variables.
var currentFilename = "";
var currentFileNumber = 0;
var currentBytesUnarchivedInFile = 0;
var currentBytesUnarchived = 0;
var totalUncompressedBytesInArchive = 0;
var totalFilesInArchive = 0;
// Helper functions.
var info = function(str) {
postMessage(new bitjs.archive.UnarchiveInfoEvent(str));
};
var err = function(str) {
postMessage(new bitjs.archive.UnarchiveErrorEvent(str));
};
var postProgress = function() {
postMessage(new bitjs.archive.UnarchiveProgressEvent(
currentFilename,
currentFileNumber,
currentBytesUnarchivedInFile,
currentBytesUnarchived,
totalUncompressedBytesInArchive,
totalFilesInArchive));
};
// Removes all characters from the first zero-byte in the string onwards.
var readCleanString = function(bstr, numBytes) {
var str = bstr.readString(numBytes);
var zIndex = str.indexOf(String.fromCharCode(0));
return zIndex != -1 ? str.substr(0, zIndex) : str;
};
// takes a ByteStream and parses out the local file information
var TarLocalFile = function(bstream) {
this.isValid = false;
// Read in the header block
this.name = readCleanString(bstream, 100);
this.mode = readCleanString(bstream, 8);
this.uid = readCleanString(bstream, 8);
this.gid = readCleanString(bstream, 8);
this.size = parseInt(readCleanString(bstream, 12), 8);
this.mtime = readCleanString(bstream, 12);
this.chksum = readCleanString(bstream, 8);
this.typeflag = readCleanString(bstream, 1);
this.linkname = readCleanString(bstream, 100);
this.maybeMagic = readCleanString(bstream, 6);
if (this.maybeMagic == "ustar") {
this.version = readCleanString(bstream, 2);
this.uname = readCleanString(bstream, 32);
this.gname = readCleanString(bstream, 32);
this.devmajor = readCleanString(bstream, 8);
this.devminor = readCleanString(bstream, 8);
this.prefix = readCleanString(bstream, 155);
if (this.prefix.length) {
this.name = this.prefix + this.name;
}
bstream.readBytes(12); // 512 - 500
} else {
bstream.readBytes(255); // 512 - 257
}
// Done header, now rest of blocks are the file contents.
this.filename = this.name;
this.fileData = null;
info("Untarring file '" + this.filename + "'");
info(" size = " + this.size);
info(" typeflag = " + this.typeflag);
// A regular file.
if (this.typeflag == 0) {
info(" This is a regular file.");
var sizeInBytes = parseInt(this.size);
this.fileData = new Uint8Array(bstream.bytes.buffer, bstream.ptr, this.size);
if (this.name.length > 0 && this.size > 0 && this.fileData && this.fileData.buffer) {
this.isValid = true;
}
bstream.readBytes(this.size);
// Round up to 512-byte blocks.
var remaining = 512 - this.size % 512;
if (remaining > 0 && remaining < 512) {
bstream.readBytes(remaining);
}
} else if (this.typeflag == 5) {
info(" This is a directory.")
}
};
// Takes an ArrayBuffer of a tar file in
// returns null on error
// returns an array of DecompressedFile objects on success
var untar = function(arrayBuffer) {
currentFilename = "";
currentFileNumber = 0;
currentBytesUnarchivedInFile = 0;
currentBytesUnarchived = 0;
totalUncompressedBytesInArchive = 0;
totalFilesInArchive = 0;
postMessage(new bitjs.archive.UnarchiveStartEvent());
var bstream = new bitjs.io.ByteStream(arrayBuffer);
var localFiles = [];
// While we don't encounter an empty block, keep making TarLocalFiles.
while (bstream.peekNumber(4) != 0) {
var oneLocalFile = new TarLocalFile(bstream);
if (oneLocalFile && oneLocalFile.isValid) {
localFiles.push(oneLocalFile);
totalUncompressedBytesInArchive += oneLocalFile.size;
}
}
totalFilesInArchive = localFiles.length;
// got all local files, now sort them
localFiles.sort(function(a,b) {
var aname = a.filename.toLowerCase();
var bname = b.filename.toLowerCase();
return aname > bname ? 1 : -1;
});
// report # files and total length
if (localFiles.length > 0) {
postProgress();
}
// now do the shipping of each file
for (var i = 0; i < localFiles.length; ++i) {
var localfile = localFiles[i];
info("Sending file '" + localfile.filename + "' up");
// update progress
currentFilename = localfile.filename;
currentFileNumber = i;
currentBytesUnarchivedInFile = localfile.size;
currentBytesUnarchived += localfile.size;
postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
postProgress();
}
postProgress();
postMessage(new bitjs.archive.UnarchiveFinishEvent());
};
// event.data.file has the ArrayBuffer.
onmessage = function(event) {
var ab = event.data.file;
untar(ab);
};
This diff is collapsed.
......@@ -76,8 +76,7 @@
<label for="config_mature_content_tags">{{_('Tags for Mature Content')}}</label>
<input type="text" class="form-control" name="config_mature_content_tags" id="config_mature_content_tags"
value="{% if content.config_mature_content_tags != None%}{{ content.config_mature_content_tags }}{% endif %}"
autocomplete="off"
>
autocomplete="off">
</div>
<div class="form-group">
<label for="config_log_level">{{_('Log Level')}}</label>
......@@ -88,6 +87,12 @@
<option value="40" {% if content.config_log_level == 40 %}selected{% endif %}>ERROR</option>
</select>
</div>
{% if rarfile_support %}
<div class="form-group">
<label for="config_rarfile_location">{{_('Location of Unrar binary')}}</label>
<input type="text" class="form-control" name="config_rarfile_location" id="config_rarfile_location" value="{% if content.config_rarfile_location != None %}{{ content.config_rarfile_location }}{% endif %}" autocomplete="off">
</div>
{% endif %}
<div class="form-group">
<input type="checkbox" id="config_uploading" name="config_uploading" {% if content.config_uploading %}checked{% endif %}>
<label for="config_uploading">{{_('Enable uploading')}}</label>
......
......@@ -13,11 +13,11 @@
<script src="{{ url_for('static', filename='js/libs/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/kthoom.js') }}"></script>
<script src="{{ url_for('static', filename='js/archive.js') }}"></script>
<!--script src="{{ url_for('static', filename='js/archive.js') }}"></script-->
<script>
document.onreadystatechange = function () {
if (document.readyState == "complete") {
init("{{ url_for('static', filename=comicfile) }}");
init("{{ url_for('get_comic_book', book_id=comicfile, book_format=extension, page=0) }}");
}
};
</script>
......
......@@ -296,6 +296,7 @@ class Settings(Base):
config_goodreads_api_key = Column(String)
config_goodreads_api_secret = Column(String)
config_mature_content_tags = Column(String) # type: str
config_rarfile_location = Column(String)
def __repr__(self):
pass
......@@ -357,6 +358,7 @@ class Config:
self.config_goodreads_api_key = data.config_goodreads_api_key
self.config_goodreads_api_secret = data.config_goodreads_api_secret
self.config_mature_content_tags = data.config_mature_content_tags
self.config_rarfile_location = data.config_rarfile_location
@property
def get_main_dir(self):
......@@ -487,6 +489,13 @@ def migrate_Database():
conn = engine.connect()
conn.execute("ALTER TABLE book_shelf_link ADD column 'order' INTEGER DEFAULT 1")
session.commit()
try:
session.query(exists().where(Settings.config_rarfile_location)).scalar()
session.commit()
except exc.OperationalError: # Database is not compatible, some rows are missing
conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_rarfile_location` String DEFAULT ''")
session.commit()
try:
create = False
session.query(exists().where(User.sidebar_view)).scalar()
......
This diff is collapsed.
......@@ -13,3 +13,4 @@ six==1.10.0
uritemplate==3.0.0
goodreads>=0.3.2
python-Levenshtein>=0.12.0
rarfile>=2.7
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment