Skip to content

Commit

Permalink
Item13100: yes, it was double-encoding, and a couple of other minor b…
Browse files Browse the repository at this point in the history 
…uglettes too
  • Loading branch information
Crawford Currie committed Jul 1, 2015
1 parent abdeb2e commit ce81eb7
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 59 deletions.
158 changes: 100 additions & 58 deletions core/tools/bulk_copy.pl
View file
Expand Up @@ -38,12 +38,11 @@
use Pod::Usage ();
use File::Spec ();
use JSON ();
use Encode ();

use version;
our $VERSION = version->declare("v1.0");

our $json = JSON->new->allow_nonref->convert_blessed(1);
our $json = JSON->new->utf8(1)->allow_nonref->convert_blessed(1);
our $session;

our %control = (
Expand All @@ -53,64 +52,99 @@
xtopic => [], # List of topics to exclude
latest => [], # List of topics for which only the latest is to be copied
quiet => 0, # shhhh
debug => 0, # OMG
check_only => 0, # If true, don't copy, just check
);

# Charset used on this site (sender and receiver may differ)
our $site_charset;
sub debug {
print STDERR "$Foswiki::VERSION: ", @_, "\n" if $control{debug};
}

#######################################################################
# Sender

sub announce {
print shift . "\n"
unless $control{quiet};
print @_, "\n" unless $control{quiet} && !$control{debug};
}

# Decode parameters from the site charset. Recurses into arrays
# and hashes, and maps references to scalars to the undecoded scalar
# value.
sub decode_cs {
my $item = shift;
# Convert a structure to/from the site charset. Recurses into arrays
# and hashes. To support binary data, it maps references to scalars
# to the *unconverted* scalar. Only required for Foswiki < 2
# $item - the thing to convert
# $action='from_unicode' - will convert a unicode structure to {Site}{CharSet}
# $action='to_unicode' - will convert a {Site}{CharSet} structure to unicode
# return the converted structure
sub convert {
my ( $item, $action ) = @_;

my $res;
if ( ref($item) eq 'HASH' ) {
$res = {};
while ( my ( $k, $v ) = each %$item ) {
$res->{ decode_cs($k) } = decode_cs($v);
$res->{ convert( $k, $action ) } = convert( $v, $action );
}
}
elsif ( ref($item) eq 'ARRAY' ) {
$res = [];
foreach my $e (@$item) {
push( @$res, decode_cs($e) );
push( @$res, convert( $e, $action ) );
}
}
elsif ( ref($item) ) {
elsif ( ref($item) eq 'SCALAR' ) {

# reference to scalar
$res = $$item;
}
elsif ( !$Foswiki::UNICODE ) {
elsif ( ref($item) ) {

# CODE, REF, GLOB, LVALUE, FORMAT, IO, VSTRING, Regexp
Carp::confess "Don't know how to convert a " . ref($item);
}
else {
my $site_charset = $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1';
eval {
$res = Encode::decode( $site_charset, $item, Encode::FB_CROAK );
require Encode;
if ( $action eq 'from_unicode' ) {

# Encoding in a site charset that may not support the
# source character. Map to HTML entities, as that is the
# most appropriate for .txt
$res =
Encode::encode( $site_charset, $item, Encode::FB_HTMLCREF );
}
else {
# $action eq 'to_unicode'
$res = Encode::decode( $site_charset, $item, Encode::FB_CROAK );
}
};
Carp::confess("$site_charset $@") if $@;
}
else {
$res = $item;
}
return $res;
}

# RPC call to a function in the receiver. Scalar parameters are
# unencoded from the site charset. Other parameters are left
# untouched.
# RPC to a function in the receiver.
sub call {
my $p = decode_cs( \@_ );
my $p = \@_;

# Decode from {Site}{CharSet} to unicode, if necessary
$p = convert( $p, 'to_unicode' ) unless $Foswiki::UNICODE;

# Encode to utf8-encoded JSON
my $json_text = $json->encode($p);
print TO_B Encode::encode_utf8($json_text) . "\n";

print TO_B "$json_text\n";

$/ = "\n";
my $response = <FROM_B>;
return undef unless defined $response;
return $json->decode($response);

# Response is utf8-encoded JSON
$response = $json->decode($response);

# Convert to {Site}{CharSet}, if necessary
$response = convert( $response, 'from_unicode' ) unless $Foswiki::UNICODE;

return $response;
}

# Copy all matched webs. This is all webs in the install, filtered
Expand Down Expand Up @@ -273,28 +307,34 @@ sub copy_topic {

# Dispatch a function call in the receiver
sub dispatch {
my $json_text = shift;
my $data = $json->decode($json_text);
my $message = shift; # a utf8-encoded JSON string

$message =~ /^(.*)$/; # untaint
my $data = $json->decode($1);

# $data is a structure containing unicode strings
return 0 unless $data;
my $fn = shift(@$data);
my @p;
eval {
@p = map {
ref($_)
? $_
: Encode::encode( $site_charset, $_, Encode::FB_HTMLCREF )
} @$data;
};
if ($@) {
}

# Convert to {Site}{Charset}, if necessary
$data = convert( $data, 'from_unicode' ) unless $Foswiki::UNICODE;
return 0 unless $data;

my $fn = shift(@$data); # function name

no strict 'refs';
my $response = &$fn(@p);
my $response = &$fn(@$data);
use strict 'refs';
$response = 0 unless defined $response;

#print STDERR "$fn(".join(',', @$data).") -> $response\n";
print TO_A $json->encode($response) . "\n";
# Convert response to unicode (if necessary)
$response = convert( $response, 'to_unicode' ) if $Foswiki::UNICODE;

$response = $json->encode($response);
debug "$fn(", join( ', ', @$data ), ") -> $response";

# JSON encode and print
print TO_A "$response\n";

return 1;
}

Expand Down Expand Up @@ -359,7 +399,7 @@ sub saveTopicRev {
dontlog => 1,
minor => 1,

# Don't call handlers (works on 1.2+ only)
# Don't call handlers (works on 2+ only)
nohandlers => 1
);
}
Expand All @@ -381,7 +421,7 @@ sub saveAttachmentRev {
forcedate => $info->{date},
stream => $fh,

# Don't call handlers (works on 1.2+ only)
# Don't call handlers (works on 2+ only)
nohandlers => 1
);
}
Expand Down Expand Up @@ -435,6 +475,7 @@ sub wildcard2regex {
'xtopic=s@' => $control{xtopic},
'latest=s@' => $control{latest},
'quietly' => sub { $control{quiet} = 1 },
'debug' => sub { $control{debug} = 1 },
'check' => sub { $control{check_only} = 1 },
'help' => sub {
Pod::Usage::pod2usage( -exitstatus => 0, -verbose => 2 );
Expand Down Expand Up @@ -490,11 +531,6 @@ sub wildcard2regex {

$session = Foswiki->new();

$site_charset =
$Foswiki::UNICODE
? 'utf-8'
: ( $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1' );

copy_webs();

print TO_B "0\n";
Expand All @@ -521,14 +557,9 @@ sub wildcard2regex {

$session = Foswiki->new();

$site_charset =
$Foswiki::UNICODE
? 'utf-8'
: ( $Foswiki::cfg{Site}{CharSet} || 'iso-8859-1' );

$/ = "\n";
while ( my $message = <FROM_A> ) {
last unless dispatch( Encode::decode_utf8($message) );
last unless dispatch($message);
$/ = "\n";
}

Expand Down Expand Up @@ -562,7 +593,7 @@ =head1 tools/bulk_copy.pl
necessary for reading/writing data to both installs (recommended that you
run it as the web user e.g. www-data)
=item B<4.> If the target installation is older than 1.2, then any plugins
=item B<4.> If the target installation is older than Foswiki 2, then any plugins
that implement before- or after- Save/Edit handlers have been disabled.
=back
Expand Down Expand Up @@ -607,6 +638,8 @@ =head1 OPTIONS
=head2 Selecting Webs and Topics
=over
=item B<--iweb> web
=item B<--xweb> web
Expand Down Expand Up @@ -651,12 +684,20 @@ =head2 Selecting Webs and Topics
any other file that has many auto-generated versions that don't
really need to be kept.)
=back
=head2 Miscellaneous
=over
=item B<--quietly>
Run quietly, without printing progress messages
=item B<--debug>
Print extra debugging information (very noisy!)
=item B<--check>
Disables the copy operations and simply runs through the two
Expand All @@ -671,18 +712,20 @@ =head2 Miscellaneous
Outputs the version number of this script.
=back
=head1 CHANGING STORES
The main purpose of this script is to support transferring Foswiki database
content between different store implementations. You might use it when
when transferring from an existing RCS-based store to a new PlainFile based
store, for example. We will use the example of upgrading an existing
Foswiki-1.1.9 installation (which uses an RCS store) to a new Foswiki-1.2.0
Foswiki-1.1.9 installation (which uses an RCS store) to a new Foswiki-2.0.0
installation (which uses PlainFile).
First, set up your two installations, so that they do not share any data
areas. You don't have to make them web-accessible. Let's say they are in
'/var/www/foswiki/Foswiki-1.1.9/bin' and '/var/www/foswiki/Foswiki-1.2.0/bin'.
'/var/www/foswiki/Foswiki-1.1.9/bin' and '/var/www/foswiki/Foswiki-2.0.0/bin'.
Now, decide what webs need to be transferred. As a general guide, you should
*not* copy the System web, otherwise you may overwrite topics
Expand All @@ -695,4 +738,3 @@ =head1 CHANGING STORES
system can be copied this way. If you want to merge revisions in different
installations together, you will have to do that manually.
=back
2 changes: 1 addition & 1 deletion core/tools/convertTopicSettings.pl
View file
Expand Up @@ -349,7 +349,7 @@ =head1 SYNOPSIS
tools/convertTopicSettings.pl [-update] [-fixdeny] [-convert] [-all] [-verbose] [-debug] [WEB ...]
This tool should be run after a Foswiki site is upgraded to
Foswiki 1.2.0 or newer. It can also be used on a Foswik 1.1.x
Foswiki 2.0 or newer. It can also be used on a Foswik 1.1.x
installation if the PatchItem12849Contrib is installed
It performs several tasks, depending up on the selected options:
Expand Down

0 comments on commit ce81eb7

Please sign in to comment.