Commit af2e2f32 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'squashfs-updates' of git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-next

Pull squashfs updates from Phillip Lougher:
 "These patches optionally improve the multi-threading peformance of
  Squashfs by adding parallel decompression, and direct decompression
  into the page cache, eliminating an intermediate buffer (removing
  memcpy overhead and lock contention)"

* tag 'squashfs-updates' of git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-next:
  Squashfs: Check stream is not NULL in decompressor_multi.c
  Squashfs: Directly decompress into the page cache for file data
  Squashfs: Restructure squashfs_readpage()
  Squashfs: Generalise paging handling in the decompressors
  Squashfs: add multi-threaded decompression using percpu variable
  squashfs: Enhance parallel I/O
  Squashfs: Refactor decompressor interface and code
parents 8b2e9b71 ed4f381e
...@@ -25,6 +25,78 @@ config SQUASHFS ...@@ -25,6 +25,78 @@ config SQUASHFS
If unsure, say N. If unsure, say N.
choice
prompt "File decompression options"
depends on SQUASHFS
help
Squashfs now supports two options for decompressing file
data. Traditionally Squashfs has decompressed into an
intermediate buffer and then memcopied it into the page cache.
Squashfs now supports the ability to decompress directly into
the page cache.
If unsure, select "Decompress file data into an intermediate buffer"
config SQUASHFS_FILE_CACHE
bool "Decompress file data into an intermediate buffer"
help
Decompress file data into an intermediate buffer and then
memcopy it into the page cache.
config SQUASHFS_FILE_DIRECT
bool "Decompress files directly into the page cache"
help
Directly decompress file data into the page cache.
Doing so can significantly improve performance because
it eliminates a memcpy and it also removes the lock contention
on the single buffer.
endchoice
choice
prompt "Decompressor parallelisation options"
depends on SQUASHFS
help
Squashfs now supports three parallelisation options for
decompression. Each one exhibits various trade-offs between
decompression performance and CPU and memory usage.
If in doubt, select "Single threaded compression"
config SQUASHFS_DECOMP_SINGLE
bool "Single threaded compression"
help
Traditionally Squashfs has used single-threaded decompression.
Only one block (data or metadata) can be decompressed at any
one time. This limits CPU and memory usage to a minimum.
config SQUASHFS_DECOMP_MULTI
bool "Use multiple decompressors for parallel I/O"
help
By default Squashfs uses a single decompressor but it gives
poor performance on parallel I/O workloads when using multiple CPU
machines due to waiting on decompressor availability.
If you have a parallel I/O workload and your system has enough memory,
using this option may improve overall I/O performance.
This decompressor implementation uses up to two parallel
decompressors per core. It dynamically allocates decompressors
on a demand basis.
config SQUASHFS_DECOMP_MULTI_PERCPU
bool "Use percpu multiple decompressors for parallel I/O"
help
By default Squashfs uses a single decompressor but it gives
poor performance on parallel I/O workloads when using multiple CPU
machines due to waiting on decompressor availability.
This decompressor implementation uses a maximum of one
decompressor per core. It uses percpu variables to ensure
decompression is load-balanced across the cores.
endchoice
config SQUASHFS_XATTR config SQUASHFS_XATTR
bool "Squashfs XATTR support" bool "Squashfs XATTR support"
depends on SQUASHFS depends on SQUASHFS
......
...@@ -5,6 +5,11 @@ ...@@ -5,6 +5,11 @@
obj-$(CONFIG_SQUASHFS) += squashfs.o obj-$(CONFIG_SQUASHFS) += squashfs.o
squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o
squashfs-y += namei.o super.o symlink.o decompressor.o squashfs-y += namei.o super.o symlink.o decompressor.o
squashfs-$(CONFIG_SQUASHFS_FILE_CACHE) += file_cache.o
squashfs-$(CONFIG_SQUASHFS_FILE_DIRECT) += file_direct.o page_actor.o
squashfs-$(CONFIG_SQUASHFS_DECOMP_SINGLE) += decompressor_single.o
squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI) += decompressor_multi.o
squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU) += decompressor_multi_percpu.o
squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
......
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "squashfs.h" #include "squashfs.h"
#include "decompressor.h" #include "decompressor.h"
#include "page_actor.h"
/* /*
* Read the metadata block length, this is stored in the first two * Read the metadata block length, this is stored in the first two
...@@ -86,16 +87,16 @@ static struct buffer_head *get_block_length(struct super_block *sb, ...@@ -86,16 +87,16 @@ static struct buffer_head *get_block_length(struct super_block *sb,
* generated a larger block - this does occasionally happen with compression * generated a larger block - this does occasionally happen with compression
* algorithms). * algorithms).
*/ */
int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, int squashfs_read_data(struct super_block *sb, u64 index, int length,
int length, u64 *next_index, int srclength, int pages) u64 *next_index, struct squashfs_page_actor *output)
{ {
struct squashfs_sb_info *msblk = sb->s_fs_info; struct squashfs_sb_info *msblk = sb->s_fs_info;
struct buffer_head **bh; struct buffer_head **bh;
int offset = index & ((1 << msblk->devblksize_log2) - 1); int offset = index & ((1 << msblk->devblksize_log2) - 1);
u64 cur_index = index >> msblk->devblksize_log2; u64 cur_index = index >> msblk->devblksize_log2;
int bytes, compressed, b = 0, k = 0, page = 0, avail; int bytes, compressed, b = 0, k = 0, avail, i;
bh = kcalloc(((srclength + msblk->devblksize - 1) bh = kcalloc(((output->length + msblk->devblksize - 1)
>> msblk->devblksize_log2) + 1, sizeof(*bh), GFP_KERNEL); >> msblk->devblksize_log2) + 1, sizeof(*bh), GFP_KERNEL);
if (bh == NULL) if (bh == NULL)
return -ENOMEM; return -ENOMEM;
...@@ -111,9 +112,9 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, ...@@ -111,9 +112,9 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
*next_index = index + length; *next_index = index + length;
TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n", TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n",
index, compressed ? "" : "un", length, srclength); index, compressed ? "" : "un", length, output->length);
if (length < 0 || length > srclength || if (length < 0 || length > output->length ||
(index + length) > msblk->bytes_used) (index + length) > msblk->bytes_used)
goto read_failure; goto read_failure;
...@@ -145,7 +146,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, ...@@ -145,7 +146,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
TRACE("Block @ 0x%llx, %scompressed size %d\n", index, TRACE("Block @ 0x%llx, %scompressed size %d\n", index,
compressed ? "" : "un", length); compressed ? "" : "un", length);
if (length < 0 || length > srclength || if (length < 0 || length > output->length ||
(index + length) > msblk->bytes_used) (index + length) > msblk->bytes_used)
goto block_release; goto block_release;
...@@ -158,9 +159,15 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, ...@@ -158,9 +159,15 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
ll_rw_block(READ, b - 1, bh + 1); ll_rw_block(READ, b - 1, bh + 1);
} }
for (i = 0; i < b; i++) {
wait_on_buffer(bh[i]);
if (!buffer_uptodate(bh[i]))
goto block_release;
}
if (compressed) { if (compressed) {
length = squashfs_decompress(msblk, buffer, bh, b, offset, length = squashfs_decompress(msblk, bh, b, offset, length,
length, srclength, pages); output);
if (length < 0) if (length < 0)
goto read_failure; goto read_failure;
} else { } else {
...@@ -168,22 +175,20 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, ...@@ -168,22 +175,20 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
* Block is uncompressed. * Block is uncompressed.
*/ */
int in, pg_offset = 0; int in, pg_offset = 0;
void *data = squashfs_first_page(output);
for (bytes = length; k < b; k++) { for (bytes = length; k < b; k++) {
in = min(bytes, msblk->devblksize - offset); in = min(bytes, msblk->devblksize - offset);
bytes -= in; bytes -= in;
wait_on_buffer(bh[k]);
if (!buffer_uptodate(bh[k]))
goto block_release;
while (in) { while (in) {
if (pg_offset == PAGE_CACHE_SIZE) { if (pg_offset == PAGE_CACHE_SIZE) {
page++; data = squashfs_next_page(output);
pg_offset = 0; pg_offset = 0;
} }
avail = min_t(int, in, PAGE_CACHE_SIZE - avail = min_t(int, in, PAGE_CACHE_SIZE -
pg_offset); pg_offset);
memcpy(buffer[page] + pg_offset, memcpy(data + pg_offset, bh[k]->b_data + offset,
bh[k]->b_data + offset, avail); avail);
in -= avail; in -= avail;
pg_offset += avail; pg_offset += avail;
offset += avail; offset += avail;
...@@ -191,6 +196,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, ...@@ -191,6 +196,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
offset = 0; offset = 0;
put_bh(bh[k]); put_bh(bh[k]);
} }
squashfs_finish_page(output);
} }
kfree(bh); kfree(bh);
......
...@@ -56,6 +56,7 @@ ...@@ -56,6 +56,7 @@
#include "squashfs_fs.h" #include "squashfs_fs.h"
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "squashfs.h" #include "squashfs.h"
#include "page_actor.h"
/* /*
* Look-up block in cache, and increment usage count. If not in cache, read * Look-up block in cache, and increment usage count. If not in cache, read
...@@ -119,9 +120,8 @@ struct squashfs_cache_entry *squashfs_cache_get(struct super_block *sb, ...@@ -119,9 +120,8 @@ struct squashfs_cache_entry *squashfs_cache_get(struct super_block *sb,
entry->error = 0; entry->error = 0;
spin_unlock(&cache->lock); spin_unlock(&cache->lock);
entry->length = squashfs_read_data(sb, entry->data, entry->length = squashfs_read_data(sb, block, length,
block, length, &entry->next_index, &entry->next_index, entry->actor);
cache->block_size, cache->pages);
spin_lock(&cache->lock); spin_lock(&cache->lock);
...@@ -220,6 +220,7 @@ void squashfs_cache_delete(struct squashfs_cache *cache) ...@@ -220,6 +220,7 @@ void squashfs_cache_delete(struct squashfs_cache *cache)
kfree(cache->entry[i].data[j]); kfree(cache->entry[i].data[j]);
kfree(cache->entry[i].data); kfree(cache->entry[i].data);
} }
kfree(cache->entry[i].actor);
} }
kfree(cache->entry); kfree(cache->entry);
...@@ -280,6 +281,13 @@ struct squashfs_cache *squashfs_cache_init(char *name, int entries, ...@@ -280,6 +281,13 @@ struct squashfs_cache *squashfs_cache_init(char *name, int entries,
goto cleanup; goto cleanup;
} }
} }
entry->actor = squashfs_page_actor_init(entry->data,
cache->pages, 0);
if (entry->actor == NULL) {
ERROR("Failed to allocate %s cache entry\n", name);
goto cleanup;
}
} }
return cache; return cache;
...@@ -410,6 +418,7 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length) ...@@ -410,6 +418,7 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length)
int pages = (length + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; int pages = (length + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
int i, res; int i, res;
void *table, *buffer, **data; void *table, *buffer, **data;
struct squashfs_page_actor *actor;
table = buffer = kmalloc(length, GFP_KERNEL); table = buffer = kmalloc(length, GFP_KERNEL);
if (table == NULL) if (table == NULL)
...@@ -421,19 +430,28 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length) ...@@ -421,19 +430,28 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length)
goto failed; goto failed;
} }
actor = squashfs_page_actor_init(data, pages, length);
if (actor == NULL) {
res = -ENOMEM;
goto failed2;
}
for (i = 0; i < pages; i++, buffer += PAGE_CACHE_SIZE) for (i = 0; i < pages; i++, buffer += PAGE_CACHE_SIZE)
data[i] = buffer; data[i] = buffer;
res = squashfs_read_data(sb, data, block, length | res = squashfs_read_data(sb, block, length |
SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, length, pages); SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, actor);
kfree(data); kfree(data);
kfree(actor);
if (res < 0) if (res < 0)
goto failed; goto failed;
return table; return table;
failed2:
kfree(data);
failed: failed:
kfree(table); kfree(table);
return ERR_PTR(res); return ERR_PTR(res);
......
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "decompressor.h" #include "decompressor.h"
#include "squashfs.h" #include "squashfs.h"
#include "page_actor.h"
/* /*
* This file (and decompressor.h) implements a decompressor framework for * This file (and decompressor.h) implements a decompressor framework for
...@@ -37,29 +38,29 @@ ...@@ -37,29 +38,29 @@
*/ */
static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = { static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = {
NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0 NULL, NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0
}; };
#ifndef CONFIG_SQUASHFS_LZO #ifndef CONFIG_SQUASHFS_LZO
static const struct squashfs_decompressor squashfs_lzo_comp_ops = { static const struct squashfs_decompressor squashfs_lzo_comp_ops = {
NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0 NULL, NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0
}; };
#endif #endif
#ifndef CONFIG_SQUASHFS_XZ #ifndef CONFIG_SQUASHFS_XZ
static const struct squashfs_decompressor squashfs_xz_comp_ops = { static const struct squashfs_decompressor squashfs_xz_comp_ops = {
NULL, NULL, NULL, XZ_COMPRESSION, "xz", 0 NULL, NULL, NULL, NULL, XZ_COMPRESSION, "xz", 0
}; };
#endif #endif
#ifndef CONFIG_SQUASHFS_ZLIB #ifndef CONFIG_SQUASHFS_ZLIB
static const struct squashfs_decompressor squashfs_zlib_comp_ops = { static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
NULL, NULL, NULL, ZLIB_COMPRESSION, "zlib", 0 NULL, NULL, NULL, NULL, ZLIB_COMPRESSION, "zlib", 0
}; };
#endif #endif
static const struct squashfs_decompressor squashfs_unknown_comp_ops = { static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
NULL, NULL, NULL, 0, "unknown", 0 NULL, NULL, NULL, NULL, 0, "unknown", 0
}; };
static const struct squashfs_decompressor *decompressor[] = { static const struct squashfs_decompressor *decompressor[] = {
...@@ -83,10 +84,11 @@ const struct squashfs_decompressor *squashfs_lookup_decompressor(int id) ...@@ -83,10 +84,11 @@ const struct squashfs_decompressor *squashfs_lookup_decompressor(int id)
} }
void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags) static void *get_comp_opts(struct super_block *sb, unsigned short flags)
{ {
struct squashfs_sb_info *msblk = sb->s_fs_info; struct squashfs_sb_info *msblk = sb->s_fs_info;
void *strm, *buffer = NULL; void *buffer = NULL, *comp_opts;
struct squashfs_page_actor *actor = NULL;
int length = 0; int length = 0;
/* /*
...@@ -94,23 +96,46 @@ void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags) ...@@ -94,23 +96,46 @@ void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags)
*/ */
if (SQUASHFS_COMP_OPTS(flags)) { if (SQUASHFS_COMP_OPTS(flags)) {
buffer = kmalloc(PAGE_CACHE_SIZE, GFP_KERNEL); buffer = kmalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
if (buffer == NULL) if (buffer == NULL) {
return ERR_PTR(-ENOMEM); comp_opts = ERR_PTR(-ENOMEM);
goto out;
}
actor = squashfs_page_actor_init(&buffer, 1, 0);
if (actor == NULL) {
comp_opts = ERR_PTR(-ENOMEM);
goto out;
}
length = squashfs_read_data(sb, &buffer, length = squashfs_read_data(sb,
sizeof(struct squashfs_super_block), 0, NULL, sizeof(struct squashfs_super_block), 0, NULL, actor);
PAGE_CACHE_SIZE, 1);
if (length < 0) { if (length < 0) {
strm = ERR_PTR(length); comp_opts = ERR_PTR(length);
goto finished; goto out;
} }
} }
strm = msblk->decompressor->init(msblk, buffer, length); comp_opts = squashfs_comp_opts(msblk, buffer, length);
finished: out:
kfree(actor);
kfree(buffer); kfree(buffer);
return comp_opts;
}
void *squashfs_decompressor_setup(struct super_block *sb, unsigned short flags)
{
struct squashfs_sb_info *msblk = sb->s_fs_info;
void *stream, *comp_opts = get_comp_opts(sb, flags);
if (IS_ERR(comp_opts))
return comp_opts;
stream = squashfs_decompressor_create(msblk, comp_opts);
if (IS_ERR(stream))
kfree(comp_opts);
return strm; return stream;
} }
...@@ -24,28 +24,22 @@ ...@@ -24,28 +24,22 @@
*/ */
struct squashfs_decompressor { struct squashfs_decompressor {
void *(*init)(struct squashfs_sb_info *, void *, int); void *(*init)(struct squashfs_sb_info *, void *);
void *(*comp_opts)(struct squashfs_sb_info *, void *, int);
void (*free)(void *); void (*free)(void *);
int (*decompress)(struct squashfs_sb_info *, void **, int (*decompress)(struct squashfs_sb_info *, void *,
struct buffer_head **, int, int, int, int, int); struct buffer_head **, int, int, int,
struct squashfs_page_actor *);
int id; int id;
char *name; char *name;
int supported; int supported;
}; };
static inline void squashfs_decompressor_free(struct squashfs_sb_info *msblk, static inline void *squashfs_comp_opts(struct squashfs_sb_info *msblk,
void *s) void *buff, int length)
{ {
if (msblk->decompressor) return msblk->decompressor->comp_opts ?
msblk->decompressor->free(s); msblk->decompressor->comp_opts(msblk, buff, length) : NULL;
}
static inline int squashfs_decompress(struct squashfs_sb_info *msblk,
void **buffer, struct buffer_head **bh, int b, int offset, int length,
int srclength, int pages)
{
return msblk->decompressor->decompress(msblk, buffer, bh, b, offset,
length, srclength, pages);
} }
#ifdef CONFIG_SQUASHFS_XZ #ifdef CONFIG_SQUASHFS_XZ
......
/*
* Copyright (c) 2013
* Minchan Kim <minchan@kernel.org>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/types.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/buffer_head.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/cpumask.h>
#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "decompressor.h"
#include "squashfs.h"
/*
* This file implements multi-threaded decompression in the
* decompressor framework
*/
/*
* The reason that multiply two is that a CPU can request new I/O
* while it is waiting previous request.
*/
#define MAX_DECOMPRESSOR (num_online_cpus() * 2)
int squashfs_max_decompressors(void)
{
return MAX_DECOMPRESSOR;
}
struct squashfs_stream {
void *comp_opts;
struct list_head strm_list;
struct mutex mutex;
int avail_decomp;
wait_queue_head_t wait;
};
struct decomp_stream {
void *stream;
struct list_head list;
};
static void put_decomp_stream(struct decomp_stream *decomp_strm,
struct squashfs_stream *stream)
{
mutex_lock(&stream->mutex);
list_add(&decomp_strm->list, &stream->strm_list);
mutex_unlock(&stream->mutex);
wake_up(&stream->wait);
}
void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
void *comp_opts)
{
struct squashfs_stream *stream;
struct decomp_stream *decomp_strm = NULL;
int err = -ENOMEM;
stream = kzalloc(sizeof(*stream), GFP_KERNEL);
if (!stream)
goto out;
stream->comp_opts = comp_opts;
mutex_init(&stream->mutex);
INIT_LIST_HEAD(&stream->strm_list);
init_waitqueue_head(&stream->wait);
/*
* We should have a decompressor at least as default
* so if we fail to allocate new decompressor dynamically,
* we could always fall back to default decompressor and
* file system works.
*/
decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
if (!decomp_strm)
goto out;
decomp_strm->stream = msblk->decompressor->init(msblk,
stream->comp_opts);
if (IS_ERR(decomp_strm->stream)) {
err = PTR_ERR(decomp_strm->stream);
goto out;
}
list_add(&decomp_strm->list, &stream->strm_list);
stream->avail_decomp = 1;
return stream;
out:
kfree(decomp_strm);
kfree(stream);
return ERR_PTR(err);
}
void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
{
struct squashfs_stream *stream = msblk->stream;
if (stream) {
struct decomp_stream *decomp_strm;
while (!list_empty(&stream->strm_list)) {
decomp_strm = list_entry(stream->strm_list.prev,
struct decomp_stream, list);
list_del(&decomp_strm->list);
msblk->decompressor->free(decomp_strm->stream);
kfree(decomp_strm);
stream->avail_decomp--;
}
WARN_ON(stream->avail_decomp);
kfree(stream->comp_opts);
kfree(stream);
}
}
static struct decomp_stream *get_decomp_stream(struct squashfs_sb_info *msblk,
struct squashfs_stream *stream)
{
struct decomp_stream *decomp_strm;
while (1) {
mutex_lock(&stream->mutex);
/* There is available decomp_stream */
if (!list_empty(&stream->strm_list)) {
decomp_strm = list_entry(stream->strm_list.prev,
struct decomp_stream, list);
list_del(&decomp_strm->list);
mutex_unlock(&stream->mutex);
break;
}
/*
* If there is no available decomp and already full,
* let's wait for releasing decomp from other users.
*/
if (stream->avail_decomp >= MAX_DECOMPRESSOR)
goto wait;
/* Let's allocate new decomp */
decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
if (!decomp_strm)
goto wait;
decomp_strm->stream = msblk->decompressor->init(msblk,
stream->comp_opts);
if (IS_ERR(decomp_strm->stream)) {
kfree(decomp_strm);
goto wait;
}
stream->avail_decomp++;
WARN_ON(stream->avail_decomp > MAX_DECOMPRESSOR);
mutex_unlock(&stream->mutex);
break;
wait:
/*
* If system memory is tough, let's for other's
* releasing instead of hurting VM because it could
* make page cache thrashing.
*/
mutex_unlock(&stream->mutex);
wait_event(stream->wait,
!list_empty(&stream->strm_list));
}
return decomp_strm;
}
int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
int b, int offset, int length, struct squashfs_page_actor *output)
{
int res;
struct squashfs_stream *stream = msblk->stream;
struct decomp_stream *decomp_stream = get_decomp_stream(msblk, stream);
res = msblk->decompressor->decompress(msblk, decomp_stream->stream,
bh, b, offset, length, output);
put_decomp_stream(decomp_stream, stream);
if (res < 0)
ERROR("%s decompression failed, data probably corrupt\n",
msblk->decompressor->name);
return res;
}
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/percpu.h>
#include <linux/buffer_head.h>
#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "decompressor.h"
#include "squashfs.h"
/*
* This file implements multi-threaded decompression using percpu
* variables, one thread per cpu core.
*/
struct squashfs_stream {
void *stream;
};
void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
void *comp_opts)
{
struct squashfs_stream *stream;
struct squashfs_stream __percpu *percpu;
int err, cpu;
percpu = alloc_percpu(struct squashfs_stream);
if (percpu == NULL)
return ERR_PTR(-ENOMEM);
for_each_possible_cpu(cpu) {
stream = per_cpu_ptr(percpu, cpu);
stream->stream = msblk->decompressor->init(msblk, comp_opts);
if (IS_ERR(stream->stream)) {
err = PTR_ERR(stream->stream);
goto out;
}
}
kfree(comp_opts);
return (__force void *) percpu;
out:
for_each_possible_cpu(cpu) {
stream = per_cpu_ptr(percpu, cpu);
if (!IS_ERR_OR_NULL(stream->stream))
msblk->decompressor->free(stream->stream);
}
free_percpu(percpu);
return ERR_PTR(err);
}
void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
{
struct squashfs_stream __percpu *percpu =
(struct squashfs_stream __percpu *) msblk->stream;
struct squashfs_stream *stream;
int cpu;
if (msblk->stream) {
for_each_possible_cpu(cpu) {
stream = per_cpu_ptr(percpu, cpu);
msblk->decompressor->free(stream->stream);
}
free_percpu(percpu);
}
}
int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
int b, int offset, int length, struct squashfs_page_actor *output)
{
struct squashfs_stream __percpu *percpu =
(struct squashfs_stream __percpu *) msblk->stream;
struct squashfs_stream *stream = get_cpu_ptr(percpu);
int res = msblk->decompressor->decompress(msblk, stream->stream, bh, b,
offset, length, output);
put_cpu_ptr(stream);
if (res < 0)
ERROR("%s decompression failed, data probably corrupt\n",
msblk->decompressor->name);
return res;
}
int squashfs_max_decompressors(void)
{
return num_possible_cpus();
}
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/types.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/buffer_head.h>
#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "decompressor.h"
#include "squashfs.h"
/*
* This file implements single-threaded decompression in the
* decompressor framework
*/
struct squashfs_stream {
void *stream;
struct mutex mutex;
};
void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
void *comp_opts)
{
struct squashfs_stream *stream;
int err = -ENOMEM;
stream = kmalloc(sizeof(*stream), GFP_KERNEL);
if (stream == NULL)
goto out;
stream->stream = msblk->decompressor->init(msblk, comp_opts);
if (IS_ERR(stream->stream)) {
err = PTR_ERR(stream->stream);
goto out;
}
kfree(comp_opts);
mutex_init(&stream->mutex);
return stream;
out:
kfree(stream);
return ERR_PTR(err);
}
void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
{
struct squashfs_stream *stream = msblk->stream;
if (stream) {
msblk->decompressor->free(stream->stream);
kfree(stream);
}
}
int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
int b, int offset, int length, struct squashfs_page_actor *output)
{
int res;
struct squashfs_stream *stream = msblk->stream;
mutex_lock(&stream->mutex);
res = msblk->decompressor->decompress(msblk, stream->stream, bh, b,
offset, length, output);
mutex_unlock(&stream->mutex);
if (res < 0)
ERROR("%s decompression failed, data probably corrupt\n",
msblk->decompressor->name);
return res;
}
int squashfs_max_decompressors(void)
{
return 1;
}
...@@ -370,77 +370,15 @@ static int read_blocklist(struct inode *inode, int index, u64 *block) ...@@ -370,77 +370,15 @@ static int read_blocklist(struct inode *inode, int index, u64 *block)
return le32_to_cpu(size); return le32_to_cpu(size);
} }
/* Copy data into page cache */
static int squashfs_readpage(struct file *file, struct page *page) void squashfs_copy_cache(struct page *page, struct squashfs_cache_entry *buffer,
int bytes, int offset)
{ {
struct inode *inode = page->mapping->host; struct inode *inode = page->mapping->host;
struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
int bytes, i, offset = 0, sparse = 0;
struct squashfs_cache_entry *buffer = NULL;
void *pageaddr; void *pageaddr;
int i, mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1;
int mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1; int start_index = page->index & ~mask, end_index = start_index | mask;
int index = page->index >> (msblk->block_log - PAGE_CACHE_SHIFT);
int start_index = page->index & ~mask;
int end_index = start_index | mask;
int file_end = i_size_read(inode) >> msblk->block_log;
TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n",
page->index, squashfs_i(inode)->start);
if (page->index >= ((i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
PAGE_CACHE_SHIFT))
goto out;
if (index < file_end || squashfs_i(inode)->fragment_block ==
SQUASHFS_INVALID_BLK) {
/*
* Reading a datablock from disk. Need to read block list
* to get location and block size.
*/
u64 block = 0;
int bsize = read_blocklist(inode, index, &block);
if (bsize < 0)
goto error_out;
if (bsize == 0) { /* hole */
bytes = index == file_end ?
(i_size_read(inode) & (msblk->block_size - 1)) :
msblk->block_size;
sparse = 1;
} else {
/*
* Read and decompress datablock.
*/
buffer = squashfs_get_datablock(inode->i_sb,
block, bsize);
if (buffer->error) {
ERROR("Unable to read page, block %llx, size %x"
"\n", block, bsize);
squashfs_cache_put(buffer);
goto error_out;
}
bytes = buffer->length;
}
} else {
/*
* Datablock is stored inside a fragment (tail-end packed
* block).
*/
buffer = squashfs_get_fragment(inode->i_sb,
squashfs_i(inode)->fragment_block,
squashfs_i(inode)->fragment_size);
if (buffer->error) {
ERROR("Unable to read page, block %llx, size %x\n",
squashfs_i(inode)->fragment_block,
squashfs_i(inode)->fragment_size);
squashfs_cache_put(buffer);
goto error_out;
}
bytes = i_size_read(inode) & (msblk->block_size - 1);
offset = squashfs_i(inode)->fragment_offset;
}
/* /*
* Loop copying datablock into pages. As the datablock likely covers * Loop copying datablock into pages. As the datablock likely covers
...@@ -451,7 +389,7 @@ static int squashfs_readpage(struct file *file, struct page *page) ...@@ -451,7 +389,7 @@ static int squashfs_readpage(struct file *file, struct page *page)
for (i = start_index; i <= end_index && bytes > 0; i++, for (i = start_index; i <= end_index && bytes > 0; i++,
bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) { bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) {
struct page *push_page; struct page *push_page;
int avail = sparse ? 0 : min_t(int, bytes, PAGE_CACHE_SIZE); int avail = buffer ? min_t(int, bytes, PAGE_CACHE_SIZE) : 0;
TRACE("bytes %d, i %d, available_bytes %d\n", bytes, i, avail); TRACE("bytes %d, i %d, available_bytes %d\n", bytes, i, avail);
...@@ -475,10 +413,74 @@ static int squashfs_readpage(struct file *file, struct page *page) ...@@ -475,10 +413,74 @@ static int squashfs_readpage(struct file *file, struct page *page)
if (i != page->index) if (i != page->index)
page_cache_release(push_page); page_cache_release(push_page);
} }
}
/* Read datablock stored packed inside a fragment (tail-end packed block) */
static int squashfs_readpage_fragment(struct page *page)
{
struct inode *inode = page->mapping->host;
struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
struct squashfs_cache_entry *buffer = squashfs_get_fragment(inode->i_sb,
squashfs_i(inode)->fragment_block,
squashfs_i(inode)->fragment_size);
int res = buffer->error;
if (res)
ERROR("Unable to read page, block %llx, size %x\n",
squashfs_i(inode)->fragment_block,
squashfs_i(inode)->fragment_size);
else
squashfs_copy_cache(page, buffer, i_size_read(inode) &
(msblk->block_size - 1),
squashfs_i(inode)->fragment_offset);
if (!sparse)
squashfs_cache_put(buffer); squashfs_cache_put(buffer);
return res;
}
static int squashfs_readpage_sparse(struct page *page, int index, int file_end)
{
struct inode *inode = page->mapping->host;
struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
int bytes = index == file_end ?
(i_size_read(inode) & (msblk->block_size - 1)) :
msblk->block_size;
squashfs_copy_cache(page, NULL, bytes, 0);
return 0;
}
static int squashfs_readpage(struct file *file, struct page *page)
{
struct inode *inode = page->mapping->host;
struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
int index = page->index >> (msblk->block_log - PAGE_CACHE_SHIFT);
int file_end = i_size_read(inode) >> msblk->block_log;
int res;
void *pageaddr;
TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n",
page->index, squashfs_i(inode)->start);
if (page->index >= ((i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
PAGE_CACHE_SHIFT))
goto out;
if (index < file_end || squashfs_i(inode)->fragment_block ==
SQUASHFS_INVALID_BLK) {
u64 block = 0;
int bsize = read_blocklist(inode, index, &block);
if (bsize < 0)
goto error_out;
if (bsize == 0)
res = squashfs_readpage_sparse(page, index, file_end);
else
res = squashfs_readpage_block(page, block, bsize);
} else
res = squashfs_readpage_fragment(page);
if (!res)
return 0; return 0;
error_out: error_out:
......
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/fs.h>
#include <linux/vfs.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/pagemap.h>
#include <linux/mutex.h>
#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "squashfs_fs_i.h"
#include "squashfs.h"
/* Read separately compressed datablock and memcopy into page cache */
int squashfs_readpage_block(struct page *page, u64 block, int bsize)
{
struct inode *i = page->mapping->host;
struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb,
block, bsize);
int res = buffer->error;
if (res)
ERROR("Unable to read page, block %llx, size %x\n", block,
bsize);
else
squashfs_copy_cache(page, buffer, buffer->length, 0);
squashfs_cache_put(buffer);
return res;
}
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/fs.h>
#include <linux/vfs.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/pagemap.h>
#include <linux/mutex.h>
#include "squashfs_fs.h"
#include "squashfs_fs_sb.h"
#include "squashfs_fs_i.h"
#include "squashfs.h"
#include "page_actor.h"
static int squashfs_read_cache(struct page *target_page, u64 block, int bsize,
int pages, struct page **page);
/* Read separately compressed datablock directly into page cache */
int squashfs_readpage_block(struct page *target_page, u64 block, int bsize)
{
struct inode *inode = target_page->mapping->host;
struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
int file_end = (i_size_read(inode) - 1) >> PAGE_CACHE_SHIFT;
int mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1;
int start_index = target_page->index & ~mask;
int end_index = start_index | mask;
int i, n, pages, missing_pages, bytes, res = -ENOMEM;
struct page **page;
struct squashfs_page_actor *actor;
void *pageaddr;
if (end_index > file_end)
end_index = file_end;
pages = end_index - start_index + 1;
page = kmalloc(sizeof(void *) * pages, GFP_KERNEL);
if (page == NULL)
return res;
/*
* Create a "page actor" which will kmap and kunmap the
* page cache pages appropriately within the decompressor
*/
actor = squashfs_page_actor_init_special(page, pages, 0);
if (actor == NULL)
goto out;
/* Try to grab all the pages covered by the Squashfs block */
for (missing_pages = 0, i = 0, n = start_index; i < pages; i++, n++) {
page[i] = (n == target_page->index) ? target_page :
grab_cache_page_nowait(target_page->mapping, n);
if (page[i] == NULL) {
missing_pages++;
continue;
}
if (PageUptodate(page[i])) {
unlock_page(page[i]);
page_cache_release(page[i]);
page[i] = NULL;
missing_pages++;
}
}
if (missing_pages) {
/*
* Couldn't get one or more pages, this page has either
* been VM reclaimed, but others are still in the page cache
* and uptodate, or we're racing with another thread in
* squashfs_readpage also trying to grab them. Fall back to
* using an intermediate buffer.
*/
res = squashfs_read_cache(target_page, block, bsize, pages,
page);
goto out;
}
/* Decompress directly into the page cache buffers */
res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor);
if (res < 0)
goto mark_errored;
/* Last page may have trailing bytes not filled */
bytes = res % PAGE_CACHE_SIZE;
if (bytes) {
pageaddr = kmap_atomic(page[pages - 1]);
memset(pageaddr + bytes, 0, PAGE_CACHE_SIZE - bytes);
kunmap_atomic(pageaddr);
}
/* Mark pages as uptodate, unlock and release */
for (i = 0; i < pages; i++) {
flush_dcache_page(page[i]);
SetPageUptodate(page[i]);
unlock_page(page[i]);
if (page[i] != target_page)
page_cache_release(page[i]);
}
kfree(actor);
kfree(page);
return 0;
mark_errored:
/* Decompression failed, mark pages as errored. Target_page is
* dealt with by the caller
*/
for (i = 0; i < pages; i++) {
if (page[i] == target_page)
continue;
flush_dcache_page(page[i]);
SetPageError(page[i]);
unlock_page(page[i]);
page_cache_release(page[i]);
}
out:
kfree(actor);
kfree(page);
return res;
}
static int squashfs_read_cache(struct page *target_page, u64 block, int bsize,
int pages, struct page **page)
{
struct inode *i = target_page->mapping->host;
struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb,
block, bsize);
int bytes = buffer->length, res = buffer->error, n, offset = 0;
void *pageaddr;
if (res) {
ERROR("Unable to read page, block %llx, size %x\n", block,
bsize);
goto out;
}
for (n = 0; n < pages && bytes > 0; n++,
bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) {
int avail = min_t(int, bytes, PAGE_CACHE_SIZE);
if (page[n] == NULL)
continue;
pageaddr = kmap_atomic(page[n]);
squashfs_copy_data(pageaddr, buffer, offset, avail);
memset(pageaddr + avail, 0, PAGE_CACHE_SIZE - avail);
kunmap_atomic(pageaddr);
flush_dcache_page(page[n]);
SetPageUptodate(page[n]);
unlock_page(page[n]);
if (page[n] != target_page)
page_cache_release(page[n]);
}
out:
squashfs_cache_put(buffer);
return res;
}
...@@ -31,13 +31,14 @@ ...@@ -31,13 +31,14 @@
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "squashfs.h" #include "squashfs.h"
#include "decompressor.h" #include "decompressor.h"
#include "page_actor.h"
struct squashfs_lzo { struct squashfs_lzo {
void *input; void *input;
void *output; void *output;
}; };
static void *lzo_init(struct squashfs_sb_info *msblk, void *buff, int len) static void *lzo_init(struct squashfs_sb_info *msblk, void *buff)
{ {
int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE);
...@@ -74,22 +75,16 @@ static void lzo_free(void *strm) ...@@ -74,22 +75,16 @@ static void lzo_free(void *strm)
} }
static int lzo_uncompress(struct squashfs_sb_info *msblk, void **buffer, static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm,
struct buffer_head **bh, int b, int offset, int length, int srclength, struct buffer_head **bh, int b, int offset, int length,
int pages) struct squashfs_page_actor *output)
{ {
struct squashfs_lzo *stream = msblk->stream; struct squashfs_lzo *stream = strm;
void *buff = stream->input; void *buff = stream->input, *data;
int avail, i, bytes = length, res; int avail, i, bytes = length, res;
size_t out_len = srclength; size_t out_len = output->length;
mutex_lock(&msblk->read_data_mutex);
for (i = 0; i < b; i++) { for (i = 0; i < b; i++) {
wait_on_buffer(bh[i]);
if (!buffer_uptodate(bh[i]))
goto block_release;
avail = min(bytes, msblk->devblksize - offset); avail = min(bytes, msblk->devblksize - offset);
memcpy(buff, bh[i]->b_data + offset, avail); memcpy(buff, bh[i]->b_data + offset, avail);
buff += avail; buff += avail;
...@@ -104,24 +99,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void **buffer, ...@@ -104,24 +99,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void **buffer,
goto failed; goto failed;
res = bytes = (int)out_len; res = bytes = (int)out_len;
for (i = 0, buff = stream->output; bytes && i < pages; i++) { data = squashfs_first_page(output);
avail = min_t(int, bytes, PAGE_CACHE_SIZE); buff = stream->output;
memcpy(buffer[i], buff, avail); while (data) {
buff += avail; if (bytes <= PAGE_CACHE_SIZE) {
bytes -= avail; memcpy(data, buff, bytes);
break;
} else {
memcpy(data, buff, PAGE_CACHE_SIZE);
buff += PAGE_CACHE_SIZE;
bytes -= PAGE_CACHE_SIZE;
data = squashfs_next_page(output);
} }
}
squashfs_finish_page(output);
mutex_unlock(&msblk->read_data_mutex);
return res; return res;
block_release:
for (; i < b; i++)
put_bh(bh[i]);
failed: failed:
mutex_unlock(&msblk->read_data_mutex);
ERROR("lzo decompression failed, data probably corrupt\n");
return -EIO; return -EIO;
} }
......
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include "page_actor.h"
/*
* This file contains implementations of page_actor for decompressing into
* an intermediate buffer, and for decompressing directly into the
* page cache.
*
* Calling code should avoid sleeping between calls to squashfs_first_page()
* and squashfs_finish_page().
*/
/* Implementation of page_actor for decompressing into intermediate buffer */
static void *cache_first_page(struct squashfs_page_actor *actor)
{
actor->next_page = 1;
return actor->buffer[0];
}
static void *cache_next_page(struct squashfs_page_actor *actor)
{
if (actor->next_page == actor->pages)
return NULL;
return actor->buffer[actor->next_page++];
}
static void cache_finish_page(struct squashfs_page_actor *actor)
{
/* empty */
}
struct squashfs_page_actor *squashfs_page_actor_init(void **buffer,
int pages, int length)
{
struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
if (actor == NULL)
return NULL;
actor->length = length ? : pages * PAGE_CACHE_SIZE;
actor->buffer = buffer;
actor->pages = pages;
actor->next_page = 0;
actor->squashfs_first_page = cache_first_page;
actor->squashfs_next_page = cache_next_page;
actor->squashfs_finish_page = cache_finish_page;
return actor;
}
/* Implementation of page_actor for decompressing directly into page cache. */
static void *direct_first_page(struct squashfs_page_actor *actor)
{
actor->next_page = 1;
return actor->pageaddr = kmap_atomic(actor->page[0]);
}
static void *direct_next_page(struct squashfs_page_actor *actor)
{
if (actor->pageaddr)
kunmap_atomic(actor->pageaddr);
return actor->pageaddr = actor->next_page == actor->pages ? NULL :
kmap_atomic(actor->page[actor->next_page++]);
}
static void direct_finish_page(struct squashfs_page_actor *actor)
{
if (actor->pageaddr)
kunmap_atomic(actor->pageaddr);
}
struct squashfs_page_actor *squashfs_page_actor_init_special(struct page **page,
int pages, int length)
{
struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
if (actor == NULL)
return NULL;
actor->length = length ? : pages * PAGE_CACHE_SIZE;
actor->page = page;
actor->pages = pages;
actor->next_page = 0;
actor->pageaddr = NULL;
actor->squashfs_first_page = direct_first_page;
actor->squashfs_next_page = direct_next_page;
actor->squashfs_finish_page = direct_finish_page;
return actor;
}
#ifndef PAGE_ACTOR_H
#define PAGE_ACTOR_H
/*
* Copyright (c) 2013
* Phillip Lougher <phillip@squashfs.org.uk>
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*/
#ifndef CONFIG_SQUASHFS_FILE_DIRECT
struct squashfs_page_actor {
void **page;
int pages;
int length;
int next_page;
};
static inline struct squashfs_page_actor *squashfs_page_actor_init(void **page,
int pages, int length)
{
struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
if (actor == NULL)
return NULL;
actor->length = length ? : pages * PAGE_CACHE_SIZE;
actor->page = page;
actor->pages = pages;
actor->next_page = 0;
return actor;
}
static inline void *squashfs_first_page(struct squashfs_page_actor *actor)
{
actor->next_page = 1;
return actor->page[0];
}
static inline void *squashfs_next_page(struct squashfs_page_actor *actor)
{
return actor->next_page == actor->pages ? NULL :
actor->page[actor->next_page++];
}
static inline void squashfs_finish_page(struct squashfs_page_actor *actor)
{
/* empty */
}
#else
struct squashfs_page_actor {
union {
void **buffer;
struct page **page;
};
void *pageaddr;
void *(*squashfs_first_page)(struct squashfs_page_actor *);
void *(*squashfs_next_page)(struct squashfs_page_actor *);
void (*squashfs_finish_page)(struct squashfs_page_actor *);
int pages;
int length;
int next_page;
};
extern struct squashfs_page_actor *squashfs_page_actor_init(void **, int, int);
extern struct squashfs_page_actor *squashfs_page_actor_init_special(struct page
**, int, int);
static inline void *squashfs_first_page(struct squashfs_page_actor *actor)
{
return actor->squashfs_first_page(actor);
}
static inline void *squashfs_next_page(struct squashfs_page_actor *actor)
{
return actor->squashfs_next_page(actor);
}
static inline void squashfs_finish_page(struct squashfs_page_actor *actor)
{
actor->squashfs_finish_page(actor);
}
#endif
#endif
...@@ -28,8 +28,8 @@ ...@@ -28,8 +28,8 @@
#define WARNING(s, args...) pr_warning("SQUASHFS: "s, ## args) #define WARNING(s, args...) pr_warning("SQUASHFS: "s, ## args)
/* block.c */ /* block.c */
extern int squashfs_read_data(struct super_block *, void **, u64, int, u64 *, extern int squashfs_read_data(struct super_block *, u64, int, u64 *,
int, int); struct squashfs_page_actor *);
/* cache.c */ /* cache.c */
extern struct squashfs_cache *squashfs_cache_init(char *, int, int); extern struct squashfs_cache *squashfs_cache_init(char *, int, int);
...@@ -48,7 +48,14 @@ extern void *squashfs_read_table(struct super_block *, u64, int); ...@@ -48,7 +48,14 @@ extern void *squashfs_read_table(struct super_block *, u64, int);
/* decompressor.c */ /* decompressor.c */
extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int); extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int);
extern void *squashfs_decompressor_init(struct super_block *, unsigned short); extern void *squashfs_decompressor_setup(struct super_block *, unsigned short);
/* decompressor_xxx.c */
extern void *squashfs_decompressor_create(struct squashfs_sb_info *, void *);
extern void squashfs_decompressor_destroy(struct squashfs_sb_info *);
extern int squashfs_decompress(struct squashfs_sb_info *, struct buffer_head **,
int, int, int, struct squashfs_page_actor *);
extern int squashfs_max_decompressors(void);
/* export.c */ /* export.c */
extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, u64, extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, u64,
...@@ -59,6 +66,13 @@ extern int squashfs_frag_lookup(struct super_block *, unsigned int, u64 *); ...@@ -59,6 +66,13 @@ extern int squashfs_frag_lookup(struct super_block *, unsigned int, u64 *);
extern __le64 *squashfs_read_fragment_index_table(struct super_block *, extern __le64 *squashfs_read_fragment_index_table(struct super_block *,
u64, u64, unsigned int); u64, u64, unsigned int);
/* file.c */
void squashfs_copy_cache(struct page *, struct squashfs_cache_entry *, int,
int);
/* file_xxx.c */
extern int squashfs_readpage_block(struct page *, u64, int);
/* id.c */ /* id.c */
extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *); extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *);
extern __le64 *squashfs_read_id_index_table(struct super_block *, u64, u64, extern __le64 *squashfs_read_id_index_table(struct super_block *, u64, u64,
......
...@@ -50,6 +50,7 @@ struct squashfs_cache_entry { ...@@ -50,6 +50,7 @@ struct squashfs_cache_entry {
wait_queue_head_t wait_queue; wait_queue_head_t wait_queue;
struct squashfs_cache *cache; struct squashfs_cache *cache;
void **data; void **data;
struct squashfs_page_actor *actor;
}; };
struct squashfs_sb_info { struct squashfs_sb_info {
...@@ -63,10 +64,9 @@ struct squashfs_sb_info { ...@@ -63,10 +64,9 @@ struct squashfs_sb_info {
__le64 *id_table; __le64 *id_table;
__le64 *fragment_index; __le64 *fragment_index;
__le64 *xattr_id_table; __le64 *xattr_id_table;
struct mutex read_data_mutex;
struct mutex meta_index_mutex; struct mutex meta_index_mutex;
struct meta_index *meta_index; struct meta_index *meta_index;
void *stream; struct squashfs_stream *stream;
__le64 *inode_lookup_table; __le64 *inode_lookup_table;
u64 inode_table; u64 inode_table;
u64 directory_table; u64 directory_table;
......
...@@ -98,7 +98,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) ...@@ -98,7 +98,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
msblk->devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE); msblk->devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE);
msblk->devblksize_log2 = ffz(~msblk->devblksize); msblk->devblksize_log2 = ffz(~msblk->devblksize);
mutex_init(&msblk->read_data_mutex);
mutex_init(&msblk->meta_index_mutex); mutex_init(&msblk->meta_index_mutex);
/* /*
...@@ -206,13 +205,14 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) ...@@ -206,13 +205,14 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
goto failed_mount; goto failed_mount;
/* Allocate read_page block */ /* Allocate read_page block */
msblk->read_page = squashfs_cache_init("data", 1, msblk->block_size); msblk->read_page = squashfs_cache_init("data",
squashfs_max_decompressors(), msblk->block_size);
if (msblk->read_page == NULL) { if (msblk->read_page == NULL) {
ERROR("Failed to allocate read_page block\n"); ERROR("Failed to allocate read_page block\n");
goto failed_mount; goto failed_mount;
} }
msblk->stream = squashfs_decompressor_init(sb, flags); msblk->stream = squashfs_decompressor_setup(sb, flags);
if (IS_ERR(msblk->stream)) { if (IS_ERR(msblk->stream)) {
err = PTR_ERR(msblk->stream); err = PTR_ERR(msblk->stream);
msblk->stream = NULL; msblk->stream = NULL;
...@@ -336,7 +336,7 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent) ...@@ -336,7 +336,7 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
squashfs_cache_delete(msblk->block_cache); squashfs_cache_delete(msblk->block_cache);
squashfs_cache_delete(msblk->fragment_cache); squashfs_cache_delete(msblk->fragment_cache);
squashfs_cache_delete(msblk->read_page); squashfs_cache_delete(msblk->read_page);
squashfs_decompressor_free(msblk, msblk->stream); squashfs_decompressor_destroy(msblk);
kfree(msblk->inode_lookup_table); kfree(msblk->inode_lookup_table);
kfree(msblk->fragment_index); kfree(msblk->fragment_index);
kfree(msblk->id_table); kfree(msblk->id_table);
...@@ -383,7 +383,7 @@ static void squashfs_put_super(struct super_block *sb) ...@@ -383,7 +383,7 @@ static void squashfs_put_super(struct super_block *sb)
squashfs_cache_delete(sbi->block_cache); squashfs_cache_delete(sbi->block_cache);
squashfs_cache_delete(sbi->fragment_cache); squashfs_cache_delete(sbi->fragment_cache);
squashfs_cache_delete(sbi->read_page); squashfs_cache_delete(sbi->read_page);
squashfs_decompressor_free(sbi, sbi->stream); squashfs_decompressor_destroy(sbi);
kfree(sbi->id_table); kfree(sbi->id_table);
kfree(sbi->fragment_index); kfree(sbi->fragment_index);
kfree(sbi->meta_index); kfree(sbi->meta_index);
......
...@@ -32,44 +32,70 @@ ...@@ -32,44 +32,70 @@
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "squashfs.h" #include "squashfs.h"
#include "decompressor.h" #include "decompressor.h"
#include "page_actor.h"
struct squashfs_xz { struct squashfs_xz {
struct xz_dec *state; struct xz_dec *state;
struct xz_buf buf; struct xz_buf buf;
}; };
struct comp_opts { struct disk_comp_opts {
__le32 dictionary_size; __le32 dictionary_size;
__le32 flags; __le32 flags;
}; };
static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff, struct comp_opts {
int len) int dict_size;
};
static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk,
void *buff, int len)
{ {
struct comp_opts *comp_opts = buff; struct disk_comp_opts *comp_opts = buff;
struct squashfs_xz *stream; struct comp_opts *opts;
int dict_size = msblk->block_size; int err = 0, n;
int err, n;
opts = kmalloc(sizeof(*opts), GFP_KERNEL);
if (opts == NULL) {
err = -ENOMEM;
goto out2;
}
if (comp_opts) { if (comp_opts) {
/* check compressor options are the expected length */ /* check compressor options are the expected length */
if (len < sizeof(*comp_opts)) { if (len < sizeof(*comp_opts)) {
err = -EIO; err = -EIO;
goto failed; goto out;
} }
dict_size = le32_to_cpu(comp_opts->dictionary_size); opts->dict_size = le32_to_cpu(comp_opts->dictionary_size);
/* the dictionary size should be 2^n or 2^n+2^(n+1) */ /* the dictionary size should be 2^n or 2^n+2^(n+1) */
n = ffs(dict_size) - 1; n = ffs(opts->dict_size) - 1;
if (dict_size != (1 << n) && dict_size != (1 << n) + if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) +
(1 << (n + 1))) { (1 << (n + 1))) {
err = -EIO; err = -EIO;
goto failed; goto out;
}
} }
} else
/* use defaults */
opts->dict_size = max_t(int, msblk->block_size,
SQUASHFS_METADATA_SIZE);
dict_size = max_t(int, dict_size, SQUASHFS_METADATA_SIZE); return opts;
out:
kfree(opts);
out2:
return ERR_PTR(err);
}
static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff)
{
struct comp_opts *comp_opts = buff;
struct squashfs_xz *stream;
int err;
stream = kmalloc(sizeof(*stream), GFP_KERNEL); stream = kmalloc(sizeof(*stream), GFP_KERNEL);
if (stream == NULL) { if (stream == NULL) {
...@@ -77,7 +103,7 @@ static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff, ...@@ -77,7 +103,7 @@ static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff,
goto failed; goto failed;
} }
stream->state = xz_dec_init(XZ_PREALLOC, dict_size); stream->state = xz_dec_init(XZ_PREALLOC, comp_opts->dict_size);
if (stream->state == NULL) { if (stream->state == NULL) {
kfree(stream); kfree(stream);
err = -ENOMEM; err = -ENOMEM;
...@@ -103,43 +129,38 @@ static void squashfs_xz_free(void *strm) ...@@ -103,43 +129,38 @@ static void squashfs_xz_free(void *strm)
} }
static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer, static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm,
struct buffer_head **bh, int b, int offset, int length, int srclength, struct buffer_head **bh, int b, int offset, int length,
int pages) struct squashfs_page_actor *output)
{ {
enum xz_ret xz_err; enum xz_ret xz_err;
int avail, total = 0, k = 0, page = 0; int avail, total = 0, k = 0;
struct squashfs_xz *stream = msblk->stream; struct squashfs_xz *stream = strm;
mutex_lock(&msblk->read_data_mutex);
xz_dec_reset(stream->state); xz_dec_reset(stream->state);
stream->buf.in_pos = 0; stream->buf.in_pos = 0;
stream->buf.in_size = 0; stream->buf.in_size = 0;
stream->buf.out_pos = 0; stream->buf.out_pos = 0;
stream->buf.out_size = PAGE_CACHE_SIZE; stream->buf.out_size = PAGE_CACHE_SIZE;
stream->buf.out = buffer[page++]; stream->buf.out = squashfs_first_page(output);
do { do {
if (stream->buf.in_pos == stream->buf.in_size && k < b) { if (stream->buf.in_pos == stream->buf.in_size && k < b) {
avail = min(length, msblk->devblksize - offset); avail = min(length, msblk->devblksize - offset);
length -= avail; length -= avail;
wait_on_buffer(bh[k]);
if (!buffer_uptodate(bh[k]))
goto release_mutex;
stream->buf.in = bh[k]->b_data + offset; stream->buf.in = bh[k]->b_data + offset;
stream->buf.in_size = avail; stream->buf.in_size = avail;
stream->buf.in_pos = 0; stream->buf.in_pos = 0;
offset = 0; offset = 0;
} }
if (stream->buf.out_pos == stream->buf.out_size if (stream->buf.out_pos == stream->buf.out_size) {
&& page < pages) { stream->buf.out = squashfs_next_page(output);
stream->buf.out = buffer[page++]; if (stream->buf.out != NULL) {
stream->buf.out_pos = 0; stream->buf.out_pos = 0;
total += PAGE_CACHE_SIZE; total += PAGE_CACHE_SIZE;
} }
}
xz_err = xz_dec_run(stream->state, &stream->buf); xz_err = xz_dec_run(stream->state, &stream->buf);
...@@ -147,23 +168,14 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer, ...@@ -147,23 +168,14 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer,
put_bh(bh[k++]); put_bh(bh[k++]);
} while (xz_err == XZ_OK); } while (xz_err == XZ_OK);
if (xz_err != XZ_STREAM_END) { squashfs_finish_page(output);
ERROR("xz_dec_run error, data probably corrupt\n");
goto release_mutex;
}
if (k < b) {
ERROR("xz_uncompress error, input remaining\n");
goto release_mutex;
}
total += stream->buf.out_pos; if (xz_err != XZ_STREAM_END || k < b)
mutex_unlock(&msblk->read_data_mutex); goto out;
return total;
release_mutex: return total + stream->buf.out_pos;
mutex_unlock(&msblk->read_data_mutex);
out:
for (; k < b; k++) for (; k < b; k++)
put_bh(bh[k]); put_bh(bh[k]);
...@@ -172,6 +184,7 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer, ...@@ -172,6 +184,7 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer,
const struct squashfs_decompressor squashfs_xz_comp_ops = { const struct squashfs_decompressor squashfs_xz_comp_ops = {
.init = squashfs_xz_init, .init = squashfs_xz_init,
.comp_opts = squashfs_xz_comp_opts,
.free = squashfs_xz_free, .free = squashfs_xz_free,
.decompress = squashfs_xz_uncompress, .decompress = squashfs_xz_uncompress,
.id = XZ_COMPRESSION, .id = XZ_COMPRESSION,
......
...@@ -32,8 +32,9 @@ ...@@ -32,8 +32,9 @@
#include "squashfs_fs_sb.h" #include "squashfs_fs_sb.h"
#include "squashfs.h" #include "squashfs.h"
#include "decompressor.h" #include "decompressor.h"
#include "page_actor.h"
static void *zlib_init(struct squashfs_sb_info *dummy, void *buff, int len) static void *zlib_init(struct squashfs_sb_info *dummy, void *buff)
{ {
z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL); z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL);
if (stream == NULL) if (stream == NULL)
...@@ -61,44 +62,37 @@ static void zlib_free(void *strm) ...@@ -61,44 +62,37 @@ static void zlib_free(void *strm)
} }
static int zlib_uncompress(struct squashfs_sb_info *msblk, void **buffer, static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm,
struct buffer_head **bh, int b, int offset, int length, int srclength, struct buffer_head **bh, int b, int offset, int length,
int pages) struct squashfs_page_actor *output)
{ {
int zlib_err, zlib_init = 0; int zlib_err, zlib_init = 0, k = 0;
int k = 0, page = 0; z_stream *stream = strm;
z_stream *stream = msblk->stream;
mutex_lock(&msblk->read_data_mutex);
stream->avail_out = 0; stream->avail_out = PAGE_CACHE_SIZE;
stream->next_out = squashfs_first_page(output);
stream->avail_in = 0; stream->avail_in = 0;
do { do {
if (stream->avail_in == 0 && k < b) { if (stream->avail_in == 0 && k < b) {
int avail = min(length, msblk->devblksize - offset); int avail = min(length, msblk->devblksize - offset);
length -= avail; length -= avail;
wait_on_buffer(bh[k]);
if (!buffer_uptodate(bh[k]))
goto release_mutex;
stream->next_in = bh[k]->b_data + offset; stream->next_in = bh[k]->b_data + offset;
stream->avail_in = avail; stream->avail_in = avail;
offset = 0; offset = 0;
} }
if (stream->avail_out == 0 && page < pages) { if (stream->avail_out == 0) {
stream->next_out = buffer[page++]; stream->next_out = squashfs_next_page(output);
if (stream->next_out != NULL)
stream->avail_out = PAGE_CACHE_SIZE; stream->avail_out = PAGE_CACHE_SIZE;
} }
if (!zlib_init) { if (!zlib_init) {
zlib_err = zlib_inflateInit(stream); zlib_err = zlib_inflateInit(stream);
if (zlib_err != Z_OK) { if (zlib_err != Z_OK) {
ERROR("zlib_inflateInit returned unexpected " squashfs_finish_page(output);
"result 0x%x, srclength %d\n", goto out;
zlib_err, srclength);
goto release_mutex;
} }
zlib_init = 1; zlib_init = 1;
} }
...@@ -109,29 +103,21 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void **buffer, ...@@ -109,29 +103,21 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void **buffer,
put_bh(bh[k++]); put_bh(bh[k++]);
} while (zlib_err == Z_OK); } while (zlib_err == Z_OK);
if (zlib_err != Z_STREAM_END) { squashfs_finish_page(output);
ERROR("zlib_inflate error, data probably corrupt\n");
goto release_mutex;
}
zlib_err = zlib_inflateEnd(stream); if (zlib_err != Z_STREAM_END)
if (zlib_err != Z_OK) { goto out;
ERROR("zlib_inflate error, data probably corrupt\n");
goto release_mutex;
}
if (k < b) { zlib_err = zlib_inflateEnd(stream);
ERROR("zlib_uncompress error, data remaining\n"); if (zlib_err != Z_OK)
goto release_mutex; goto out;
}
length = stream->total_out; if (k < b)
mutex_unlock(&msblk->read_data_mutex); goto out;
return length;
release_mutex: return stream->total_out;
mutex_unlock(&msblk->read_data_mutex);
out:
for (; k < b; k++) for (; k < b; k++)
put_bh(bh[k]); put_bh(bh[k]);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment