Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Item13516: fixed problems with bulk_copy between FW2 stores. Added di…
…fficult web/topic names to RCS test generator. Corrected doc in PFS (no functional changes)
  • Loading branch information
Crawford Currie committed Jul 20, 2015
1 parent 75c9bf5 commit 45ca0df
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 26 deletions.
38 changes: 22 additions & 16 deletions PlainFileStoreContrib/lib/Foswiki/Store/PlainFile.pm
Expand Up @@ -8,28 +8,34 @@ Single-file implementation of =Foswiki::Store= that uses normal
files in a standard directory structure to store versions.
* Webs map to directories; webs only "exist" if they contain a
preferences topic.
* Topics are in data/.../topic.txt. If there is no .txt for a topic,
the topic does not exist, even if there is a history.
* Topic histories are in data/.../topic,pfv/
WebPreferences topic.
* Latest revs for topics are in data/.../TopicName.txt. If there
is no .txt for a topic, the topic does not exist, even if there
is a history.
* Topic histories are in data/.../TopicName,pfv/
* Each rev of the topic has a numbered file containing the text of that
rev (1 2 3 etc) each with a corresponding metafile 1.m 2.m etc.
* Attachment histories are in data/.../topic,pfv/ATTACHMENTS/attachmentname/
* Each rev of an attachment has a numbered file containing the data for
that rev (same as a topic), each with a corresponding metafile
(same as a topic)
* The latest rev always has a history file (note: this means that
large attachments are stored at least twice; same as in the RCS stores)
rev (1 2 .. N), each with a corresponding metafile 1.m 2.m .. N.m
* Latest attachments are in pub/.../TopicName/attach.ment
* Attachment histories are in
data/.../topic,pfv/ATTACHMENTS/attach.ment/
* Same as a topic, each rev of an attachment has a numbered file
containing the data for that rev, each with a corresponding
metafile N.m
* The latest rev of an attachment always has a history file
(note: this means that all attachments are stored at least
twice; same as in the RCS stores)
* 'date' always comes from the file modification date
* 'author' and 'comment' come from the metafile
* 'version' comes from the name of the version file
* 'version' comes from the *name* of the version file
A note on character encodings. This module is designed to work best when
data is stored using UTF-8, but can also use an alternate encoding by
Note that .m metafiles currently only contain the CUID of the contributor.
Other metadata is stored embedded in topic text.
A note on character encodings. This store is designed to work best when
data is stored using UTF-8, but you can also use an alternate encoding by
setting {Store}{Encoding}. Conversion to/from the alternate
encoding is done at the lowest possible level - before calling file-level
operations - so in general, strings can be assumed to be UTF-8 encoded
byte strings.
operations.
NOTE: Perl's low-level file operations treat file names as sequences of
bytes. When a function such as 'open' is called and is passed a unicode
Expand Down
32 changes: 26 additions & 6 deletions RCSStoreContrib/test/unit/RCSStoreContrib/gen_bulk_copy_test.pl
Expand Up @@ -163,10 +163,11 @@ sub ci {

sub make_topic {
my ( $name, $text, $rev ) = @_;
my $path = "data/$web/$name";
my $path = "data/$web/"
. Encode::encode( $source_encoding, $name, Encode::FB_CROAK );
push( @made, "topic $name version $rev " );
open( F, ">:encoding($source_encoding)", "$path.txt" )
|| die recode( "Failed", $path, $! );
|| die recode("Failed $path $!");
print F <<THIS;
%META:TOPICINFO{author="ProjectContributor" date="$time" format="1.1" version="$rev"}%
$text
Expand All @@ -178,9 +179,12 @@ sub make_topic {

sub make_attachment {
my ( $topic, $name, $data, $rev ) = @_;
mkdir "pub/$web/$topic";
my $path = "pub/$web/$topic/$name";
open( F, ">", $path ) || die recode( "Failed", $path, $! );
my $path = "pub/$web/"
. Encode::encode( $source_encoding, $topic, Encode::FB_CROAK );
mkdir $path;
my $path =
$path . Encode::encode( $source_encoding, $name, Encode::FB_CROAK );
open( F, ">", $path ) || die recode("Failed $path $!");
binmode(F);
print F $data;
close(F);
Expand All @@ -204,7 +208,20 @@ sub make_attachment {
make_topic( "RevHistory", "REV 2", 2 );
make_topic( "RevHistory", "REV 3", 1 );

# Make a topc with attachments
# Make a web with an evil name
my $evil =
substr( $classes{upper}, -10, 5 )
. substr( $classes{lower}, -5 )
. substr( $classes{upper}, -5 );
mkdir "data/$web/$evil";
make_topic( "$evil/WebPreferences", 'REV 1', 1 );

# Make a topic with the same name
make_topic( "$evil/$evil", $evil, 1 );

# Make a web with the same name

# Make a topic with attachments
my $att_name = make_chars( 5, 'alnum' ) . '.att';
make_topic( "HasAttachments", <<CONTENT, 1 );
%META:FILEATTACHMENT{name="$att_name" comment="logo" user="ProjectContributor" version="1" date="$time"}%
Expand All @@ -231,6 +248,9 @@ sub make_attachment {
The contents should be:
$made
The evil subweb is called $evil
DATA

1;
19 changes: 15 additions & 4 deletions core/tools/bulk_copy.pl
Expand Up @@ -39,6 +39,8 @@
use Pod::Usage ();
use File::Spec ();
use JSON ();
use Error;
$Error::Debug = 1; # verbose stack traces, please

use version;
our $VERSION = version->declare("v1.0");
Expand Down Expand Up @@ -103,6 +105,11 @@ sub convert {
# CODE, REF, GLOB, LVALUE, FORMAT, IO, VSTRING, Regexp
Carp::confess "Don't know how to convert a " . ref($item);
}
elsif ($Foswiki::UNICODE) {

# Everything is unicode, only need to deref scalars
$res = $item;
}
else {
my $site_charset = $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1';
eval {
Expand Down Expand Up @@ -130,7 +137,7 @@ sub call {
my $p = \@_;

# Decode from {Site}{CharSet} to unicode, if necessary
$p = convert( $p, 'to_unicode' ) unless $Foswiki::UNICODE;
$p = convert( $p, 'to_unicode' );

# Encode to utf8-encoded JSON
my $json_text = $json->encode($p);
Expand All @@ -147,7 +154,7 @@ sub call {
$response = $json->decode($response);

# Convert to {Site}{CharSet}, if necessary
$response = convert( $response, 'from_unicode' ) unless $Foswiki::UNICODE;
$response = convert( $response, 'from_unicode' );

die $response if $status;

Expand Down Expand Up @@ -393,7 +400,7 @@ sub dispatch {
return 0 unless $data;

# Convert to {Site}{Charset}, if necessary
$data = convert( $data, 'from_unicode' ) unless $Foswiki::UNICODE;
$data = convert( $data, 'from_unicode' );
return 0 unless $data;

my $fn = shift(@$data); # function name
Expand All @@ -411,7 +418,7 @@ sub dispatch {
}

# Convert response to unicode (if necessary)
$response = convert( $response, 'to_unicode' ) if $Foswiki::UNICODE;
$response = convert( $response, 'to_unicode' );

$response = $json->encode($response);
debug "$fn(", join( ', ', @$data ), ") -> $response";
Expand Down Expand Up @@ -619,6 +626,8 @@ sub wildcard2regex {
require Foswiki;
die $@ if $@;

binmode( STDOUT, ':utf8' ) if $Foswiki::UNICODE;

announce
"Copying from $Foswiki::VERSION ($Foswiki::cfg{Store}{Implementation})";

Expand All @@ -645,6 +654,8 @@ sub wildcard2regex {
$Foswiki::cfg{Engine} = 'Foswiki::Engine::CLI';
require Foswiki;

binmode( STDOUT, ':utf8' ) if $Foswiki::UNICODE;

announce
"Copying to $Foswiki::VERSION ($Foswiki::cfg{Store}{Implementation})";

Expand Down

0 comments on commit 45ca0df

Please sign in to comment.