Note: this list is kept only as a demonstration for CVSNotice. For the latest CVS notices, see the Xaraya and Postnuke sites
View Statistics - Next Notice - Previous NoticeDirectory filter : [ all ] / postnuke_official / tests [ view in CVS ]
| Date | Directory [filter] | File(s) [view] | Author [filter] |
| 02 Aug 2002 13:14:14 | postnuke_official/tests | import8.php,1.10,1.11 | Mike |
| work in steps to import large sites (2000+ users, 1000+ articles) on slow servers | |||
Update of /home/cvsroot/postnuke_official/tests
In directory ns7.hostnuke.net:/tmp/cvs-serv25966
Modified Files:
import8.php
Log Message:
work in steps to import large sites (2000+ users, 1000+ articles) on slow servers
Index: import8.php
===================================================================
RCS file: /home/cvsroot/postnuke_official/tests/import8.php,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** import8.php 28 Jul 2002 16:05:10 -0000 1.10
--- import8.php 2 Aug 2002 13:14:12 -0000 1.11
***************
*** 28,33 ****
--- 28,35 ----
pnInit();
+ if (!isset($step)) {
// start the output buffer
ob_start();
+ }
?>
***************
*** 37,40 ****
--- 39,48 ----
global $pnconfig;
$prefix = $pnconfig['prefix'];
+ if (isset($step) && ($step > 1 || isset($startnum))) {
+ $oldprefix = pnModGetVar('import','oldprefix');
+ $reset = pnModGetVar('import','reset');
+ $resetcat = pnModGetVar('import','resetcat');
+ $imgurl = pnModGetVar('import','imgurl');
+ }
if (!isset($oldprefix) || $oldprefix == $prefix || !preg_match('/^[a-z0-9]+$/i',$oldprefix)) {
?>
***************
*** 42,81 ****
<p></p>
<form method="POST" action="import8.php">
! Prefix used in your .71 site :
! <input type="text" name="oldprefix" value="not <?php echo $prefix ?> !">
! <br><br>URL of the /images directory on your .71 site :
! <input type="text" name="imgurl" value="/images">
! <br><br><input type="checkbox" name="reset"> Reset .8 data ?
! <input type="submit" value=" Import Data ">
</form>
<p></p>
You must also have activated categories and users, and added the articles
! module from postnuke_modules.
<?php
} else {
list($dbconn) = pnDBGetConn();
$pntable = pnDBGetTables();
if (!isset($reset)) {
$reset = 0;
}
echo "<strong>1. Importing users</strong><br>\n";
! if (!pnModAPILoad('users','admin')) {
! die("Unable to load the users admin API");
}
$query = 'SELECT pn_uid, pn_name, pn_uname, pn_email, pn_pass, pn_url
FROM ' . $oldprefix . '_users
! WHERE pn_uid > 2';
! $result = $dbconn->Execute($query);
if ($dbconn->ErrorNo() != 0) {
die("Oops, select users failed : " . $dbconn->ErrorMsg());
}
! if ($reset) {
$dbconn->Execute("DELETE FROM " . $pntable['users'] . " WHERE pn_uid > 2");
}
while (!$result->EOF) {
list($uid,$name,$uname,$email,$pass,$url) = $result->fields;
! $insert = 'INSERT INTO ' . $pntable['users'] . '
(pn_uid, pn_name, pn_uname, pn_email, pn_pass, pn_url,
pn_auth_module)
--- 50,138 ----
<p></p>
<form method="POST" action="import8.php">
! <table border="0" cellpadding="4">
! <tr><td align="right">Prefix used in your .71 site</td><td>
! <input type="text" name="oldprefix" value="not '<?php echo $prefix ?>' !"></td></tr>
! <tr><td align="right">URL of the /images directory on your .71 site</td><td>
! <input type="text" name="imgurl" value="/images"></td></tr>
! <tr><td align="right">Reset corresponding .8 data ?</td><td>
! <input type="checkbox" name="reset" checked></td></tr>
! <tr><td align="right">Reset existing .8 categories ?</td><td>
! <input type="checkbox" name="resetcat" checked></td></tr>
! <tr><td colspan=2 align="middle">
! <input type="submit" value=" Import Data "></td></tr>
! </table>
! <input type="hidden" name="step" value="1">
</form>
<p></p>
You must also have activated categories and users, and added the articles
! module from postnuke_modules first.
<?php
} else {
+ if ($step == 1 && !isset($startnum)) {
+ pnModSetVar('import','oldprefix',$oldprefix);
+ if (!isset($reset)) { $reset = 0; }
+ pnModSetVar('import','reset',$reset);
+ if (!isset($resetcat)) { $resetcat = 0; }
+ pnModSetVar('import','resetcat',$resetcat);
+ if (!isset($imgurl)) { $imgurl = 0; }
+ pnModSetVar('import','imgurl',$imgurl);
+ }
+
list($dbconn) = pnDBGetConn();
$pntable = pnDBGetTables();
+ if (!pnModAPILoad('users','admin')) {
+ die("Unable to load the users admin API");
+ }
+ if (!pnModAPILoad('categories','user')) {
+ die("Unable to load the categories user API");
+ }
+ if (!pnModAPILoad('categories','admin')) {
+ die("Unable to load the categories admin API");
+ }
+ if (!pnModAPILoad('articles','admin')) {
+ die("Unable to load the articles admin API");
+ }
+
if (!isset($reset)) {
$reset = 0;
}
+ if (!isset($resetcat)) {
+ $resetcat = 0;
+ }
+ if ($step == 1) {
echo "<strong>1. Importing users</strong><br>\n";
! $query = 'SELECT COUNT(*) FROM ' . $oldprefix . '_users';
! $result = $dbconn->Execute($query);
! if ($dbconn->ErrorNo() != 0) {
! die("Oops, count users failed : " . $dbconn->ErrorMsg());
}
+ $count = $result->fields[0];
+ $result->Close();
$query = 'SELECT pn_uid, pn_name, pn_uname, pn_email, pn_pass, pn_url
FROM ' . $oldprefix . '_users
! WHERE pn_uid > 2
! ORDER BY pn_uid ASC';
! $numitems = 2000;
! if (!isset($startnum)) {
! $startnum = 0;
! }
! if ($count > $numitems) {
! $result = $dbconn->SelectLimit($query, $numitems, $startnum);
! } else {
! $result = $dbconn->Execute($query);
! }
if ($dbconn->ErrorNo() != 0) {
die("Oops, select users failed : " . $dbconn->ErrorMsg());
}
! if ($reset && $startnum == 0) {
$dbconn->Execute("DELETE FROM " . $pntable['users'] . " WHERE pn_uid > 2");
}
+ $num = 1;
while (!$result->EOF) {
list($uid,$name,$uname,$email,$pass,$url) = $result->fields;
! // $insert = 'INSERT INTO ' . $pntable['users'] . '
! $insert = 'INSERT DELAYED INTO ' . $pntable['users'] . '
(pn_uid, pn_name, pn_uname, pn_email, pn_pass, pn_url,
pn_auth_module)
***************
*** 91,112 ****
if ($dbconn->ErrorNo() != 0) {
echo "Insert user ($uid) $name failed : " . $dbconn->ErrorMsg() ."<br>\n";
! } else {
! echo "Inserted user ($uid) $name $uname $email $url<br>\n";
}
$result->MoveNext();
}
$result->Close();
echo "<strong>TODO : import user_data</strong><br><br>\n";
echo "<strong>2. Importing old news topics into categories</strong><br>\n";
! if (!pnModAPILoad('categories','user')) {
! die("Unable to load the categories user API");
}
if ($reset) {
! // $dbconn->Execute("DELETE FROM " . $pntable['categories']);
! $dbconn->Execute("DELETE FROM " . $pntable['categories_linkage'] . " WHERE pn_modid=151");
! }
! if (!pnModAPILoad('categories','admin')) {
! die("Unable to load the categories admin API");
}
echo "Creating root for old news topics<br>\n";
--- 148,179 ----
if ($dbconn->ErrorNo() != 0) {
echo "Insert user ($uid) $name failed : " . $dbconn->ErrorMsg() ."<br>\n";
! } elseif ($count < 200) {
! echo "Inserted user ($uid) $name - $uname<br>\n";
! } elseif ($num % 100 == 0) {
! echo "Inserted user " . ($num + $startnum) . "<br>\n";
! flush();
}
+ $num++;
$result->MoveNext();
}
$result->Close();
echo "<strong>TODO : import user_data</strong><br><br>\n";
+ echo '<a href="import8.php">Return to start</a> ';
+ if ($count > $numitems && $startnum + $numitems < $count) {
+ $startnum += $numitems;
+ echo '<a href="import8.php?step=' . $step . '&startnum=' . $startnum . '">Go to step ' . $step . ' - users ' . $startnum . '+ of ' . $count . '</a><br>';
+ } else {
+ echo '<a href="import8.php?step=' . ($step+1) . '">Go to step ' . ($step+1) . '</a><br>';
+ }
+ }
+ if ($step == 2) {
echo "<strong>2. Importing old news topics into categories</strong><br>\n";
! if ($resetcat) {
! $dbconn->Execute("DELETE FROM " . $pntable['categories']);
}
if ($reset) {
! $regid = pnModGetIDFromName('articles');
! $dbconn->Execute("DELETE FROM " . $pntable['categories_linkage'] . " WHERE pn_modid=$regid");
}
echo "Creating root for old news topics<br>\n";
***************
*** 122,127 ****
// preset the article categories to those two types
if ($reset) {
! pnModSetVar('articles', 'number_of_categories', 2);
! pnModSetVar('articles', 'cids', "$topics;$categories");
}
echo "Creating old default 'Articles' news category<br>\n";
--- 189,196 ----
// preset the article categories to those two types
if ($reset) {
! pnModSetVar('articles', 'number_of_categories.1', 2);
! pnModSetVar('articles', 'cids.1', "$topics;$categories");
! } else {
! // you'll be in trouble with your categories here...
}
echo "Creating old default 'Articles' news category<br>\n";
***************
*** 166,185 ****
$result->Close();
echo "<strong>TODO : copy the topic images to modules/categories/pnimages or elsewhere someday</strong><br><br>\n";
echo "<strong>3. Importing articles</strong><br>\n";
! if (!pnModAPILoad('articles','admin')) {
! die("Unable to load the articles admin API");
! }
! if ($reset) {
! $dbconn->Execute("DELETE FROM " . $pntable['articles']);
}
$regid = pnModGetIDFromName('articles');
$query = 'SELECT pn_sid, pn_title, pn_hometext, pn_bodytext, pn_aid,
UNIX_TIMESTAMP(pn_time), pn_language, pn_catid, pn_topic
FROM ' . $oldprefix . '_stories';
! $result = $dbconn->Execute($query);
if ($dbconn->ErrorNo() != 0) {
die("Oops, select stories failed : " . $dbconn->ErrorMsg());
}
while (!$result->EOF) {
list($aid, $title, $summary, $body, $authorid, $pubdate, $language,
--- 235,279 ----
$result->Close();
echo "<strong>TODO : copy the topic images to modules/categories/pnimages or elsewhere someday</strong><br><br>\n";
+ pnModSetVar('import','topics',$topics);
+ pnModSetVar('import','topicid',serialize($topicid));
+ pnModSetVar('import','categories',$categories);
+ pnModSetVar('import','catid',serialize($catid));
+ echo '<a href="import8.php">Return to start</a>
+ <a href="import8.php?step=' . ($step+1) . '">Go to step ' . ($step+1) . '</a><br>';
+ }
+ if ($step == 3) {
+ $topics = pnModGetVar('import','topics');
+ $topicid = unserialize(pnModGetVar('import','topicid'));
+ $categories = pnModGetVar('import','categories');
+ $catid = unserialize(pnModGetVar('import','catid'));
echo "<strong>3. Importing articles</strong><br>\n";
! $query = 'SELECT COUNT(*) FROM ' . $oldprefix . '_stories';
! $result = $dbconn->Execute($query);
! if ($dbconn->ErrorNo() != 0) {
! die("Oops, count stories failed : " . $dbconn->ErrorMsg());
}
+ $count = $result->fields[0];
+ $result->Close();
$regid = pnModGetIDFromName('articles');
$query = 'SELECT pn_sid, pn_title, pn_hometext, pn_bodytext, pn_aid,
UNIX_TIMESTAMP(pn_time), pn_language, pn_catid, pn_topic
FROM ' . $oldprefix . '_stories';
! $numitems = 1000;
! if (!isset($startnum)) {
! $startnum = 0;
! }
! if ($count > $numitems) {
! $result = $dbconn->SelectLimit($query, $numitems, $startnum);
! } else {
! $result = $dbconn->Execute($query);
! }
if ($dbconn->ErrorNo() != 0) {
die("Oops, select stories failed : " . $dbconn->ErrorMsg());
}
+ if ($reset && $startnum == 0) {
+ $dbconn->Execute("DELETE FROM " . $pntable['articles']);
+ }
+ $num = 1;
while (!$result->EOF) {
list($aid, $title, $summary, $body, $authorid, $pubdate, $language,
***************
*** 201,207 ****
if ($dbconn->ErrorNo() != 0) {
echo "Insert article ($aid) $title failed : " . $dbconn->ErrorMsg() ."<br>\n";
! } else {
echo "Inserted article ($aid) $title<br>\n";
}
$cids = array();
if (isset($topicid[$topic])) {
--- 295,305 ----
if ($dbconn->ErrorNo() != 0) {
echo "Insert article ($aid) $title failed : " . $dbconn->ErrorMsg() ."<br>\n";
! } elseif ($count < 200) {
echo "Inserted article ($aid) $title<br>\n";
+ } elseif ($num % 100 == 0) {
+ echo "Inserted article " . ($num + $startnum) . "<br>\n";
+ flush();
}
+ $num++;
$cids = array();
if (isset($topicid[$topic])) {
***************
*** 219,223 ****
--- 317,331 ----
$result->Close();
echo "<strong>TODO : add notes, comments etc.</strong><br><br>\n";
+ echo '<a href="import8.php">Return to start</a> ';
+ if ($count > $numitems && $startnum + $numitems < $count) {
+ $startnum += $numitems;
+ echo '<a href="import8.php?step=' . $step . '&startnum=' . $startnum . '">Go to step ' . $step . ' - articles ' . $startnum . '+ of ' . $count . '</a><br>';
+ } else {
+ // OK, we still have something to do here :-)
+ echo '<a href="import8.php?step=' . ($step+2) . '">Go to step ' . ($step+2) . '</a><br>';
+ }
+ }
+ if ($step == 5) {
echo "<strong>5. Importing old sections into categories</strong><br>\n";
echo "Creating root for old sections<br>\n";
***************
*** 226,229 ****
--- 334,341 ----
'description' => 'Root for old sections',
'parent_id' => 0));
+ if ($reset) {
+ pnModSetVar('articles', 'number_of_categories.2', 1);
+ pnModSetVar('articles', 'cids.2', $sections);
+ }
if ($sections > 0) {
$query = 'SELECT pn_secid, pn_secname, pn_image
***************
*** 246,250 ****
--- 358,371 ----
}
echo "<strong>TODO : copy the section images to modules/categories/pnimages or elsewhere someday</strong><br><br>\n";
+ pnModSetVar('import','sections',$sections);
+ pnModSetVar('import','sectionid',serialize($sectionid));
+ echo '<a href="import8.php">Return to start</a>
+ <a href="import8.php?step=' . ($step+1) . '">Go to step ' . ($step+1) . '</a><br>';
+ }
+ if ($step == 6) {
+ $regid = pnModGetIDFromName('articles');
+ $sections = pnModGetVar('import','sections');
+ $sectionid = unserialize(pnModGetVar('import','sectionid'));
echo "<strong>6. Importing section content</strong><br>\n";
$query = 'SELECT pn_artid, pn_secid, pn_title, pn_content, pn_language
***************
*** 294,298 ****
}
$result->Close();
! echo "<strong>TODO : allow to filter 'articles' on different roots (e.g. News, Sections, FAQs, ...)</strong><br><br>\n";
}
--- 415,432 ----
}
$result->Close();
! echo "<strong>TODO : import the rest...</strong><br><br>\n";
! pnModDelVar('import','oldprefix');
! pnModDelVar('import','reset');
! pnModDelVar('import','resetcat');
! pnModDelVar('import','imgurl');
! pnModDelVar('import','topics');
! pnModDelVar('import','topicid');
! pnModDelVar('import','categories');
! pnModDelVar('import','catid');
! pnModDelVar('import','sections');
! pnModDelVar('import','sectionid');
! echo '<a href="import8.php">Return to start</a>
! <a href="index.php">Go to your imported site</a><br>';
! }
}
***************
*** 300,313 ****
<?php
// catch the output
$return = ob_get_contents();
ob_end_clean();
- // close the session
- session_write_close();
-
// render the page
$regenerate = false;
pnTplPrintPage($return, pnUserGetTheme(), 'default', $regenerate);
// done
--- 434,454 ----
<?php
+ if (!isset($step)) {
+
// catch the output
$return = ob_get_contents();
ob_end_clean();
// render the page
$regenerate = false;
pnTplPrintPage($return, pnUserGetTheme(), 'default', $regenerate);
+ }
+
+ // close the session
+ session_write_close();
+
+ $dbconn->Close();
+
+ flush();
// done
View Statistics - Next Notice - Previous Notice
| Visit Developer Site - Browse CVS Repository |
Syndicate via backend.rss (max. once per hour please) | Powered by CVSNotice 0.1.3 |