'),
'#description' => t('The list of tags which are allowed in feeds, i.e., which will not be removed by Drupal.')
);
$form['feedapi_allow_html_all'] = array(
'#type' => 'checkbox', '#title' => t('Allow all HTML tags'),
'#default_value' => variable_get('feedapi_allow_html_all', FALSE),
'#description' => t('In this case the module does\'t filter any HTML elements from the incoming fields. This checkbox overrides the above list of allowed tags.')
);
if (variable_get('feedapi_allow_html_all', FALSE)) {
$form['feedapi_allowed_html_tags']['#disabled'] = TRUE;
}
// Drupal will try to overwrite this value at cron time
$max_exec = !ini_get('safe_mode') ? 240 : ini_get('max_execution_time');
$form['feedapi_cron_percentage'] = array(
'#type' => 'select',
'#title' => t('Cron time for FeedAPI [%]'),
'#options' => drupal_map_assoc(array(15, 25, 50, 75)),
'#default_value' => variable_get('feedapi_cron_percentage', 15),
'#description' => t('Percentage of maximal PHP execution time (currently !exec seconds).
At current settings, the FeedAPI cron process can run for up to !now seconds.',
array("!exec" => $max_exec, "!now" => (variable_get('feedapi_cron_percentage', 15) / 100) * $max_exec)),
);
return system_settings_form($form);
}
/**
* Create a feedapi node programatically.
*
* @param $param
* Either a feedapi - enabled node type or a $node object with at least valid $node->type.
* @param $url
* URI of feed.
*/
function feedapi_create_node($param, $url) {
if (is_object($param)) {
$node = $param;
}
else {
$node = new stdClass();
$node->type = $param;
}
if (!feedapi_enabled($node->type)) {
return FALSE;
}
$feed = _feedapi_build_feed_object($node->type, $url);
if (!$feed->title && !$node->title) {
return FALSE;
}
$node->title = $node->title ? $node->title : $feed->title;
$node->body = $node->body ? $node->body : $feed->description;
$node->feed = $feed;
$node_options = variable_get('node_options_'. $node->type, array('status', 'promote'));
// If this is a new node, fill in the default values.
foreach (array('status', 'promote', 'sticky') as $key) {
$node->$key = in_array($key, $node_options);
}
// Get the content-type settings as default
$node->feedapi = feedapi_get_settings($node->type);
node_object_prepare($node);
global $user;
$node->uid = $user->uid;
node_save($node);
return $node;
}
/**
* Load node by URL.
* @param $args
* Currently only supported $args['url] - URL string.
* @return
* Node object if successful, FALSE if not.
*/
function feedapi_load_node($args) {
if ($nid = db_result(db_query("SELECT nid FROM {feedapi} WHERE url = '%s'", $args['url']))) {
return node_load($nid);
}
return FALSE;
}
/**
* Refresh a feed node (= run enabled processors on it).
* @param $node
* A node object with a $node->feed object.
* @param $destination_path
* If a destination path is given, function redirects to this destination.
*/
function feedapi_refresh($node, $destination_path = NULL) {
feedapi_invoke_feedapi('refresh', $node->feed, FALSE);
if ($destination_path) {
drupal_goto($destination_path);
}
}
/**
* Insert feedapi data to the DB when it's a new for for FeedAPI
*/
function _feedapi_insert(&$node, $teaser, $page) {
if (isset($node->feed->url) && isset($node->feed->feed_type)) {
db_query("INSERT INTO {feedapi} (
nid, url, link, feed_type, processors,
parsers, checked, settings) VALUES
(%d, '%s', '%s', '%s', '%s', '%s', %d, '%s')",
$node->nid,
$node->feed->url,
$node->feed->options->link,
$node->feed->feed_type,
serialize($node->feed->processors),
serialize($node->feed->parsers),
0,
serialize(array())
);
// Store add on module's settings if user has permission to do so.
if (user_access('advanced feedapi options')) {
_feedapi_store_settings(array('nid' => $node->nid), $node->feedapi);
}
// Refresh feed if the user would like to do that
if ($node->feedapi['refresh_on_create'] == TRUE) {
$node->feed->nid = $node->nid;
feedapi_invoke_feedapi('refresh', $node->feed);
}
}
}
/**
* Update feed data of an existing feed
*/
function _feedapi_update(&$node, $teaser, $page) {
if (isset($node->feed)) {
$old_config = node_load($node->nid);
// In that case this feed has never have feed data. Should be created then, this is not really an update
if (!is_numeric($old_config->feed->nid)) {
$node->feed = _feedapi_build_feed_object($node->type, $node->feed->url);
_feedapi_insert($node, 'insert', $teaser, $page);
return;
}
foreach ($old_config->feed->processors as $processor) {
if (!in_array($processor, $node->feed->processors)) {
$items = module_invoke($processor, 'feedapi_item', 'fetch', $node->feed);
foreach ($items as $item) {
module_invoke($processor, 'feedapi_item', 'delete', $item);
}
}
}
db_query("UPDATE {feedapi} SET
url = '%s', feed_type = '%s', processors = '%s',
parsers = '%s', link = '%s' WHERE nid = %d",
$node->feed->url,
$node->feed->feed_type,
serialize($node->feed->processors),
serialize($node->feed->parsers),
$node->feed->options->link,
$node->nid
);
// Store add on module's settings if user has permission to do so.
if (user_access('advanced feedapi options')) {
_feedapi_store_settings(array('nid' => $node->nid), $node->feedapi);
}
}
}
/**
* Execute the enabled parsers and create an unified output
*
* @param $feed
* Feed object
* @param $parsers
* Structure: array(
* "primary" => "parser_primary",
* "secondary" => array("parser1", "parser2", "parserN")
* );
* @return
* The object of the parser data
*/
function _feedapi_call_parsers($feed, $parsers) {
$nid = $feed->nid;
$parser_primary = array_shift($parsers);
$parsers_secondary = $parsers;
if (module_exists($parser_primary)) {
$feed->feed_type = module_invoke($parser_primary, 'feedapi_feed', 'compatible', $feed);
$parser_output = module_invoke($parser_primary, 'feedapi_feed', 'parse', $feed);
if ($parser_output === FALSE) {
return $parser_output;
}
$feed = (object) array_merge((array) $feed, (array) $parser_output);
}
// Call the turned on parsers, create a union of returned options
$parsers_secondary = is_array($parsers_secondary) ? $parsers_secondary : array();
foreach ($parsers_secondary as $parser) {
$feed_ext = module_invoke($parser, 'feedapi_feed', 'parse', $feed, FALSE);
$feed->options = (object) ((array) $feed->options + (array) $feed_ext->options);
// Merge items' options
if (is_array($feed_ext->items)) {
foreach ($feed_ext->items as $key => $item) {
$feed->items[$key]->options = (object) ((array) $feed->items[$key]->options + (array) $item->options);
}
}
}
$feed->nid = $nid;
foreach (module_implements('feedapi_after_parse') as $module) {
$func = $module .'_feedapi_after_parse';
$func($feed);
}
// Filter bad or not allowed tags, sanitize data (currently timestamp checking)
if (!variable_get('feedapi_allow_html_all', FALSE)) {
$allowed = preg_split('/\s+|<|>/', variable_get('feedapi_allowed_html_tags', '
'), -1, PREG_SPLIT_NO_EMPTY);
foreach (array('title', 'description') as $property) {
if (isset($feed->{$property})) {
if (is_string($feed->{$property})) {
$feed->{$property} = filter_xss($feed->{$property}, $allowed);
}
}
}
for ($i = 0; $i < count($feed->items); $i++) {
$feed->items[$i]->title = filter_xss($feed->items[$i]->title, $allowed);
$feed->items[$i]->description = filter_xss($feed->items[$i]->description, $allowed);
if ($feed->items[$i]->options->timestamp == 0) {
$feed->items[$i]->options->timestamp = time();
}
}
}
return $feed;
}
/**
* Stores settings per content type or per node.
*
* @param $args
* Associative array which is $args['nid'] = N or $args['node_type'] = "content_type". Depends on what to store for
* @param $settings
* The settings data itself
*/
function _feedapi_store_settings($args, $settings) {
if ($args['nid']) {
db_query("UPDATE {feedapi} SET settings = '%s', skip = %d WHERE nid = %d", serialize($settings), $settings['skip'], $args['nid']);
module_invoke_all('feedapi_after_settings', $args['nid'], $settings);
// This ensures that next time, not the cached, but the updated value will be used.
static $node_settings;
unset($node_settings[$args['nid']]);
}
elseif ($args['node_type']) {
variable_set('feedapi_settings_'. $args['node_type'], $settings);
}
}
/**
* Determines wether feedapi is enabled for given node type.
* If parser or processor is passed in, this function determines wether given
* parser or processor is enabled for given node type.
* @param $node_type
* A Drupal node type.
* @param $parser_or_processor
* A parser or processor - pass in by module name.
* @return TRUE if enabled, FALSE if not.
*/
function feedapi_enabled($node_type, $parser_or_processor = '') {
$settings = feedapi_get_settings($node_type);
if (!$parser_or_processor || !$settings['enabled']) {
return $settings['enabled'] ? TRUE : FALSE;
}
foreach (array('parsers', 'processors') as $stage) {
if ($settings[$stage][$parser_or_processor]['enabled']) {
return TRUE;
}
}
return FALSE;
}
/**
* Helper function for feedapi_invoke().
*
* Generic operations, collects results and returns array
*/
function _feedapi_invoke($op, &$feed, $param) {
$output = array();
foreach ($feed->processors as $processor) {
$result = module_invoke($processor, 'feedapi_item', $op, $feed, $param);
// Result may be a list of items or single values (count)
if ($result) {
if (is_array($result)) {
$output = array_merge($output, $result);
}
else {
$output[] = $result;
}
}
}
return $output;
}
/**
* Helper function for feedapi_invoke().
* Load a list of feed items into the feed object from the processors.
*/
function _feedapi_invoke_load(&$feed, $param) {
$feed->items = array();
foreach ($feed->processors as $processor) {
$items = module_invoke($processor, 'feedapi_item', 'fetch', $feed);
if (is_array($items)) {
foreach ($items as $item) {
$feed->items[] = $item;
}
}
}
}
/**
* Helper function for feedapi_invoke().
* Refresh the feed, call the proper parsers and processors' hooks.
* Don't call this function directly, use feedapi_refresh() instead.
*
* @ TODO Fix: This may loop forever when a feed has no processors
*/
function _feedapi_invoke_refresh(&$feed, $param) {
$timestamp = variable_get('cron_semaphore', FALSE) !== FALSE ? variable_get('cron_semaphore', FALSE) : time();
$counter = array();
timer_start('feedapi_'. $feed->nid);
$cron = $param;
// Step 0: Check processors and parsers and grab settings
if (!is_array($feed->processors) || count($feed->processors) == 0) {
if (!$cron) {
drupal_set_message(t("No processors specified for URL %url. Could not refresh.", array('%url' => $feed->url)), "error");
drupal_goto('node/'. $feed->nid);
}
return 0;
}
else if (!is_array($feed->parsers) || count($feed->parsers) == 0) {
drupal_set_message(t("No parsers specified for URL %url. Could not refresh.", array('%url' => $feed->url)), "error");
drupal_goto('node/'. $feed->nid);
}
$settings = feedapi_get_settings(NULL, $feed->nid);
// Step 1: Force processors to delete old items and determine the max. create elements.
$counter['expired'] = feedapi_expire($feed);
// Step 2: Get feed.
$nid = $feed->nid;
$hash_old = $feed->hash;
$feed = _feedapi_call_parsers($feed, $feed->parsers, $feed->half_done);
if (is_object($feed)) {
$feed->hash = md5(serialize($feed->items));
}
// Step 3: See, whether feed has been modified.
if ($feed === FALSE || $hash_old == $feed->hash) {
// Updated the checked field in any case.
db_query("UPDATE {feedapi} SET checked = %d, half_done = %d WHERE nid = %d", time(), FALSE, $nid);
if (!$cron) {
if ($hash_old == $feed->hash && is_object($feed)) {
drupal_set_message(t('There are no new items in the feed.'), 'status');
}
else {
drupal_set_message(t('Could not refresh feed.'), 'error');
}
}
return $counter;
}
// Step 4: Walk through the items and check duplicates, then save or update
$items = $feed->items;
$updated = 0;
$new = 0;
$half_done = FALSE;
// We check for time-out after each item
foreach ($items as $index => $item) {
// Call each item parser
$item->is_updated = FALSE;
$item->is_new = FALSE;
foreach ($feed->processors as $processor) {
if (!module_invoke($processor, 'feedapi_item', 'unique', $item, $feed->nid, $settings['processors'][$processor])) {
if ($settings['update_existing'] == TRUE) {
module_invoke($processor, 'feedapi_item', 'update', $item, $feed->nid, $settings['processors'][$processor]);
$item->is_updated = TRUE;
}
}
else {
// We have checked before for expired items, so just save it.
// if the item is already expired then do nothing
$items_delete = $settings['items_delete'];
$diff = abs(time() - (isset($item->options->timestamp) ? $item->options->timestamp : time()));
if ($diff > $items_delete && ($items_delete > FEEDAPI_NEVER_DELETE_OLD)) {
break;
}
module_invoke($processor, 'feedapi_item', 'save', $item, $feed->nid, $settings['processors'][$processor]);
$item->is_new = TRUE;
}
}
$new = $item->is_new ? $new + 1 : $new;
$updated = ($item->is_updated && !$item->is_new) ? $updated + 1 : $updated;
// Decision on time. If the exec time is greather than the user-set percentage of php max execution time
if ($cron && !feedapi_cron_time()) {
$half_done = ($new + $updated) == count($items) ? FALSE : TRUE;
break;
}
// Save the item status for further processing
$feed->items[$index] = $item;
}
// Closing step: Call after refresh and update feed statistics
foreach (module_implements('feedapi_after_refresh') as $module) {
$func = $module .'_feedapi_after_refresh';
$func($feed);
}
if ($new > 0) {
_feedapi_store_stat($nid, 'update_times', time(), $timestamp);
_feedapi_store_stat($nid, 'new', $new, $timestamp);
_feedapi_store_stat($nid, 'download_num', count($items), $timestamp);
_feedapi_store_stat($nid, 'process_time', timer_read('feedapi_'. $feed->nid), $timestamp);
}
db_query("UPDATE {feedapi} SET checked = %d, half_done = %d, hash = '%s' WHERE nid = %d", time(), $half_done, $feed->hash, $feed->nid);
if (!$cron) {
if ($new == 0 && $updated == 0) {
drupal_set_message(t('There are no new items in the feed.'), 'status');
}
else {
drupal_set_message(t("%new new item(s) were saved. %updated existing item(s) were updated.", array("%new" => $new, "%updated" => $updated)));
}
// @ TODO what value to return here?
}
else {
// Update and return counter
$counter['new'] = $new;
$counter['updated'] = $updated;
return $counter;
}
}
/**
* Helper function for feedapi_invoke().
* Delete all feed items of a feed.
*/
function _feedapi_invoke_purge(&$feed, $param) {
$node = node_load($feed->nid);
// Reset hash.
db_query("UPDATE {feedapi} SET hash = 0 WHERE nid = %d", $feed->nid);
if ($param == 'items') {
return drupal_get_form('feedapi_purge_confirm', $node);
}
feedapi_invoke_feedapi('load', $feed);
// Delete items from the processors
foreach ($feed->items as $item) {
foreach ($feed->processors as $processor) {
// FIXME: it's possible now to accidentally delete an item from another processor
module_invoke($processor, 'feedapi_item', 'delete', $item, $feed->settings['processors'][$processor]);
}
}
if ($param == 'items_confirmed') {
drupal_set_message(t('!count feed items have been deleted successfully from the feed.', array('!count' => count($feed->items))));
}
}
/**
* Builds feed object ready to be sticked onto node.
*/
function _feedapi_build_feed_object($node_type, $url) {
$feed = new stdClass();
$feed->url = $url;
$node_type_settings = feedapi_get_settings($node_type);
$feed->processors = _feedapi_format_settings($node_type_settings, 'processors');
$feed->parsers = _feedapi_format_settings($node_type_settings, 'parsers');
if ($feed->url) {
$feed = _feedapi_call_parsers($feed, $feed->parsers);
}
$feed->link = $feed->options->link;
return $feed;
}
/**
* Returns per content type settings ordered by weight
* and only those that are turned on.
* @param $node_type_settings
* Content type settings retrieved with feedapi_get_settings().
* @param $stage_type
* 'parsers' or 'processors'
*/
function _feedapi_format_settings($node_type_settings, $stage_type) {
$result = array();
$settings = $node_type_settings[$stage_type];
if (is_array($settings)) {
foreach ($settings as $name => $properties) {
if ($settings[$name]['enabled']) {
$result[$settings[$name]['weight']] = $name;
}
}
ksort($result);
}
return $result;
}
/**
* Retrieve settings per content type or per node.
* @param $node_type
* @param $nid
* @return
* Associative array with feedapi settings
* @todo: Use node type settings for pulling on/off and weight of
* parsers/processors, use per node settings to override their
* configuration, this allows us a more predictable
* presets/settings behaviour. See d. o. #191692
* Watch out: cache permutations of node_type or node_type+nid or nid.
* Watch out: changes within page load likely.
*/
function feedapi_get_settings($node_type, $nid = FALSE) {
static $node_settings;
if (is_numeric($nid)) {
if (!isset($node_settings[$nid])) {
if ($settings = db_fetch_object(db_query('SELECT settings FROM {feedapi} WHERE nid = %d', $nid))) {
$settings = unserialize($settings->settings);
}
$node_settings[$nid] = is_array($settings) && !empty($settings) ? $settings : FALSE;
}
if (!is_array($node_settings[$nid])) {
if (empty($node_type)) {
// In normal case, this shouldn't happen. This is an emergency branch
$node_type = db_result(db_query("SELECT type FROM {node} WHERE nid = %d", $nid));
}
}
else {
return $node_settings[$nid];
}
}
// Fallback: node_type.
if (isset($node_type) && is_string($node_type)) {
if ($settings = variable_get('feedapi_settings_'. $node_type, FALSE)) {
return $settings;
}
}
return FALSE;
}
/**
* Set default value of $form elements if present in $settings.
*/
function _feedapi_populate($form, $settings) {
static $i;
$i++;
foreach ($form as $k => $v) {
if (is_array($v)) {
if (array_key_exists('#default_value', $v)) {
// Don't prepopulate feedapi_url slot, not stored in settings
// Might be overwritten otherwise by users without advanced feedapi options permissions.
// Todo: stick all settings form elements that are not in 'parsers' or 'processors' in 'general' -
// This is kind of tricky though without breaking sites out there.
if ($k != 'feedapi_url') {
$form[$k]['#default_value'] = $settings[$k];
}
}
else {
$form[$k] = _feedapi_populate($form[$k], $settings[$k]);
}
}
}
$i--;
return $form;
}
/**
* Calculate the average between-update time
*/
function _feedapi_update_rate($update_times) {
for ($i = 0; $i < count($update_times) - 1; $i++) {
$between[] = abs($update_times[$i] - $update_times[$i + 1]);
}
return (count($between) > 0) ? round(array_sum($between) / count($between), 2) : t('No data yet');
}
/**
* Remove non-existing processors from the processors arrays
*/
function _feedapi_sanitize_processors(&$feed) {
if (is_array($feed->processors)) {
foreach ($feed->processors as $key => $processor) {
if (!module_exists($processor)) {
unset($feed->processors[$key]);
}
}
}
}
/**
* Generates an OPML representation of all feeds.
*/
function _feedapi_export_opml() {
$result = db_query(db_rewrite_sql('SELECT n.title, f.url FROM {feedapi} f INNER JOIN {node} n ON f.nid = n.nid ORDER BY n.title ASC'));
$output = "\n";
$output .= "