Feeds are based on content-types. Before you can create a feed, you have to go to the admin/content/types/add page. There choose a name for the content-type, then enable the "Is a feed content type" checkbox under the Feed API group. Next choose the processors and parsers that you want to use. At least one parser and one processor must be enabled. Then create the content-type. Make sure that permissions are given to the users who will handle the feeds. '); } } /** * Implementation of hook_menu(). */ function feedapi_menu($may_cache) { if ($may_cache) { $items[] = array( 'path' => 'admin/content/feed', 'title' => t('Feeds'), 'description' => t("Overview which content your site aggregates from other sites and see detailed statistics about the feeds."), 'callback' => 'feedapi_management_page', 'access' => user_access('administer feedapi'), ); $items[] = array( 'path' => 'admin/content/feed/list', 'title' => t('List'), 'type' => MENU_DEFAULT_LOCAL_TASK, 'access' => user_access('administer feedapi'), 'weight' => -15, ); $items[] = array( 'path' => 'admin/content/feed/import_opml', 'title' => t('Import OPML'), 'type' => MENU_NORMAL_ITEM, 'access' => user_access('administer feedapi'), 'callback' => 'drupal_get_form', 'callback arguments' => array('feedapi_import_feeds_form'), ); $items[] = array( 'path' => 'admin/content/feed/export_opml', 'title' => t('Import OPML'), 'type' => MENU_CALLBACK, 'access' => user_access('administer feedapi'), 'callback' => 'drupal_get_form', 'callback arguments' => array('_feedapi_export_opml'), ); $items[] = array('path' => 'admin/settings/feedapi', 'title' => t('FeedAPI settings'), 'callback' => 'drupal_get_form', 'callback arguments' => array('feedapi_admin_settings'), 'type' => MENU_NORMAL_ITEM, 'access' => user_access('administer feedapi'), ); } else if (arg(0) == 'node' && is_numeric(arg(1))) { $node = node_load(arg(1)); if (isset($node->feed)) { global $user; $own_feed = $node->uid == $user->uid && user_access('edit own '. $node->type . ' content') ? TRUE : FALSE; $items[] = array('path' => 'node/'. $node->nid. '/refresh', 'title' => t('Refresh'), 'callback' => 'feedapi_invoke_feedapi', 'callback arguments' => array("refresh", $node->feed), 'type' => MENU_LOCAL_TASK, 'access' => variable_get('feedapi_feed_show_commands_link', TRUE) && (user_access('administer feedapi') || $own_feed), ); $items[] = array('path' => 'node/'. $node->nid. '/purge', 'title' => t('Remove items'), 'callback' => 'feedapi_invoke_feedapi', 'callback arguments' => array("purge", $node->feed, 'items'), 'type' => MENU_LOCAL_TASK, 'access' => variable_get('feedapi_feed_show_commands_link', TRUE) && (user_access('administer feedapi') || $own_feed), ); } } return $items; } /** * Implementation of hook_nodeapi(). */ function feedapi_nodeapi(&$node, $op, $teaser, $page) { if (isset($node->feed) || _feedapi_is_enabled($node->type)) { switch ($op) { case 'insert': if (isset($node->feed->url) && isset($node->feed->feed_type)) { db_query("INSERT INTO {feedapi} ( nid, url, link, feed_type, processors, parsers, checked, settings) VALUES (%d, '%s', '%s', '%s', '%s', '%s', %d, '%s')", $node->nid, $node->feed->url, $node->feed->options->link, $node->feed->feed_type, serialize($node->feed->processors), serialize($node->feed->parsers), 0, serialize(array()) ); _feedapi_store_settings(array('nid' => $node->nid), $node->feedapi); } break; case 'update': if (isset($node->feed)) { $old_config = node_load($node->nid); foreach ($old_config->feed->processors as $processor) { if (!in_array($processor, $node->feed->processors)) { $items = module_invoke($processor, 'feedapi_item', 'fetch', $node->feed); foreach ($items as $item) { module_invoke($processor, 'feedapi_item', 'delete', $item); } } } db_query("UPDATE {feedapi} SET url = '%s', feed_type = '%s', processors = '%s', parsers = '%s', link = '%s', items_delete = %d, update_existing = %d WHERE nid = %d", $node->feed->url, $node->feed->feed_type, serialize($node->feed->processors), serialize($node->feed->parsers), $node->feed->options->link, $node->feedapi['feedapi_items_delete'], $node->feedapi['feedapi_update_existing'], $node->nid ); _feedapi_store_settings(array('nid' => $node->nid), $node->feedapi); } break; case 'load': $node->feed = db_fetch_object(db_query('SELECT * FROM {feedapi} WHERE nid = %d', $node->nid)); $node->feed->settings = _feedapi_get_settings(array('nid' => $node->nid)); // ksort is because of ensure the ordering of the processors $node->feed->processors = unserialize($node->feed->processors); $node->feed->statistics = unserialize($node->feed->statistics); if (is_array($node->feed->processors)) { ksort($node->feed->processors); } else { $node->feed->processors = array(); } $node->feed->parsers = unserialize($node->feed->parsers); break; case 'delete': // Could be a performance problem - think of thousands of node feed items. feedapi_invoke_feedapi('purge', $node->feed); db_query("DELETE FROM {feedapi} WHERE nid = %d", $node->nid); break; case 'validate': if (!isset($node->nid) && !empty($node->feedapi['feedapi_url'])) { if (db_result(db_query("SELECT COUNT(*) FROM {feedapi} WHERE url = '%s'", $node->feedapi['feedapi_url'])) > 0) { form_set_error('feedapi_url', t('A feed from this URL (%url) already exists.', array('%url' => $node->feedapi['feedapi_url']))); } $feed = new stdClass(); $feed->url = $node->feedapi['feedapi_url']; $settings = _feedapi_get_settings(array('node_type' => $node->type)); $assigned = _feedapi_assign_parser_processor($feed->url, $settings); $feed->processors = $assigned['processors']; $feed->parsers = $assigned['parsers']; $feed->feed_type = $assigned['type']; if (empty($feed->feed_type)) { form_set_error('feedapi', t('Error parsing feed.')); } else { $feed = _feedapi_call_parsers($feed, $feed->parsers, FALSE); } if (!$feed->title) { form_set_error('feedapi_url', t('Feed information could not be retrieved.')); } } break; case 'submit': $feed = new stdClass(); $feed->url = $node->feedapi['feedapi_url']; $assigned = _feedapi_assign_parser_processor($feed->url, _feedapi_get_settings(array('node_type' => $node->type))); $feed->processors = $assigned['processors']; $feed->parsers = $assigned['parsers']; $feed->feed_type = $assigned['type']; $feed = _feedapi_call_parsers($feed, $feed->parsers, FALSE); $feed->link = $feed->options->link; $node->title = empty($node->title) ? $feed->title : $node->title; $node->body = empty($node->body) ? $feed->description : $node->body; $node->feed->options->link = $feed->options->link; $node->feed->url = $feed->url; $node->feed = $feed; break; } } } /** * Implementation of hook_node_type(). */ function feedapi_node_type($op, $info) { switch ($op){ case 'delete': variable_del('feedapi_settings_'. $info->type); // Todo: find out, why and where there is still feedapi_$info->type being stored. variable_del('feedapi_'. $info->type); break; case 'update': if (!empty($info->old_type) && $info->old_type != $info->type) { $setting = variable_get('feedapi_settings_'. $info->old_type, array()); variable_del('feedapi_settings_'. $info->old_type); variable_set('feedapi_settings_'. $info->type, $setting); } break; } } /** * Implementation of hook_block(). */ function feedapi_block($op = 'list', $delta = 0) { $types = array_keys(node_get_types()); foreach ($types as $key => $type) { if (!_feedapi_is_enabled($type)) { unset($types[$key]); } } switch ($op) { case 'list': foreach ($types as $type) { $blocks[$type]['info'] = t('FeedAPI: Create feed with !preset preset', array('!preset' => $type)); } return $blocks; case 'view': $block['subject'] = t('Create feed with !preset preset', array('!preset' => $delta)); $block['content'] = drupal_get_form('feedapi_simplified_form', $delta); return $block; } } /** * Implementation of hook_perm(). */ function feedapi_perm() { return array('administer feedapi', 'advanced feedapi options'); } /** * Implementation of hook_link(). */ function feedapi_link($type, $node = NULL) { if ($type == 'node' && isset($node->feed)) { if (variable_get('feedapi_show_feed_origin_link', TRUE) && strlen($node->feed->link) > 0) { $links['feedapi_original'] = array( 'title' => t('Link to site'), 'href' => $node->feed->link, ); return $links; } } } /** * Implementation of hook_simpletest(). */ function feedapi_simpletest() { $tests = file_scan_directory(drupal_get_path('module', 'feedapi'), '\.test'); return array_keys($tests); } /** * Do various things with feed. Handle the core data and call the * underyling modules (parsers/processors) too. * TODO: Clean up this function, break it in smaller pieces that are * easier to handle. * * @param $op * "load" Load the feed items basic data into the $feed->items[] * "refresh" Re-download the feed and process newly arrived item * * @param $feed * A feed object. If only the ID is known, you should pass something like this: $feed->nid = X * @param $param * Depends on the $op value. */ function feedapi_invoke_feedapi($op, &$feed, $param = NULL) { if (!is_object($feed)) { return FALSE; } if (!isset($feed->processors)) { $node = node_load($feed->nid); if (!isset($node->feed)) { return FALSE; } $feed = $node->feed; } _feedapi_sanitize_processors($feed); switch ($op) { case 'load': $feed->items = array(); foreach ($feed->processors as $weight) { foreach ($weight as $processor) { $items = module_invoke($processor, 'feedapi_item', 'fetch', $feed); if (is_array($items)) { foreach ($items as $item) { $feed->items[] = $item; } } } } break; case 'refresh': timer_start('feedapi_'. $feed->nid); $cron = $param; if (!is_array($feed->processors) || count($feed->processors) == 0) { if (!$cron) { drupal_set_message(t("This feed (%url) doesn't have any processor. It's not possible to refresh the feed.", array('%url' => $feed->url)), "error"); drupal_goto('node/'. $feed->nid); } return; } $processors = $feed->processors; // Force the processors to delete old items and determine the max. create elements $refresh = feedapi_expire($feed, $cron === TRUE ? FALSE : TRUE); if ($refresh == FALSE) { return; } $nid = $feed->nid; // force non-caching mode if the feed is only processed half-done $feed = _feedapi_call_parsers($feed, $feed->parsers, $feed->half_done ? FALSE : TRUE); if ($feed === FALSE) { // Refresh the Last refresh field at all the cases db_query("UPDATE {feedapi} SET checked = %d, half_done = %d WHERE nid = %d", time(), FALSE, $nid); if (!$cron) { drupal_set_message(t('This feed has not been refreshed after the last check.')); drupal_goto('node/'. $nid); } return; } $feed->processors = $processors; $items = array_reverse($feed->items); $updated = 0; $new = 0; $half_done = FALSE; // Walk through the items foreach ($items as $item) { // Call each item parser $is_updated = FALSE; $is_new = FALSE; foreach ($feed->processors as $weight) { foreach ($weight as $processor) { if (!module_invoke($processor, 'feedapi_item', 'unique', $item, $feed->nid, $feed->settings['processors'][$processor])) { if ($feed->update_existing == TRUE) { module_invoke($processor, 'feedapi_item', 'update', $item, $feed->nid, $feed->settings['processors'][$processor]); $is_updated = TRUE; } } else { module_invoke($processor, 'feedapi_item', 'save', $item, $feed->nid, $feed->settings['processors'][$processor]); $is_new = TRUE; } } } $new = $is_new ? $new + 1 : $new; $updated = ($is_updated && !$is_new) ? $updated + 1 : $updated; // decision on time. If the exec time is greather than the user-set percentage of php max execution time if ($cron && ((timer_read('feedapi_cron') / 1000) > ((variable_get('feedapi_cron_percentage', 15) / 100) * ini_get('max_execution_time')))) { $timeout = TRUE; $half_done = ($new + $updated) == count($items) ? FALSE : TRUE; break; } } foreach (module_implements('feedapi_after_refresh') as $module) { $func = $module. '_feedapi_after_refresh'; $func($feed); } if ($new > 0) { $feed->statistics['update_times'] = _feedapi_queue($feed->statistics['update_times'], time()); $feed->statistics['new'] = _feedapi_queue($feed->statistics['new'], $new); $feed->statistics['download_num'] = _feedapi_queue($feed->statistics['download_num'], count($items)); $feed->statistics['process_time'] = _feedapi_queue($feed->statistics['process_time'], timer_read('feedapi_'. $feed->nid)); } db_query("UPDATE {feedapi} SET checked = %d, statistics = '%s', half_done = %d WHERE nid = %d", time(), serialize($feed->statistics), $half_done, $feed->nid); if (!$cron) { drupal_set_message(t("%new new item(s) were saved. %updated existing item(s) were updated.", array("%new" => $new, "%updated" => $updated))); drupal_goto('node/'. $nid); } if ($timeout === TRUE) { return FALSE; } break; case 'purge': $node = node_load($feed->nid); if ($param == 'items') { return drupal_get_form('feedapi_purge_confirm', $node); } feedapi_invoke_feedapi('load', $feed); // Delete items from the processors foreach ($feed->items as $item) { foreach($feed->processors as $weight) { foreach ($weight as $processor) { // FIXME: it's possible now to delete an item from another processor as an incident module_invoke($processor, 'feedapi_item', 'delete', $item, $feed->settings['processors'][$processor]); } } } if ($param == 'items_confirmed') { drupal_set_message(t('!count feed items have been deleted successfully from the feed.', array('!count' => count($feed->items)))); } break; } } /** * Ask for confirmation before deletig all the items */ function feedapi_purge_confirm($node) { $output = confirm_form( array('nid' => array('#type' => hidden, '#value' => $node->nid)), t('Delete all the feed items from !name', array('!name' => $node->title)), 'node/'. $node->nid, t("Are you sure you want to delete all the feed items from !name?", array('!name' => $node->title)), t('Yes'), t('No'), 'feedapi_purge_confirm' ); return $output; } /** * Submitted items purging form. Drop all the items. */ function feedapi_purge_confirm_submit($form_id, $form_values) { $feed->nid = $form_values['nid']; feedapi_invoke_feedapi('purge', $feed, 'items_confirmed'); drupal_goto('node/'. $form_values['nid']); } /** * OPML Feed import form, also allows setting defaults to be applied to each feed */ function feedapi_import_feeds_form() { global $user; $form['#attributes'] = array('enctype' => 'multipart/form-data'); $form['opml'] = array( '#type' => 'file', '#title' => t('OPML File'), '#size' => 50, '#description' => t('Upload an OPML file containing a list of newsfeeds to be imported.'), ); $nt = node_get_types(); foreach ($nt as $type => $info) { if (variable_get('feedapi_settings_'. $type, FALSE) !== FALSE) { $feed_types[$type] = $info->name; } } $form['feed_type'] = array( '#type' => 'select', '#title' => t('Feed Type'), '#description' => t("The type of feed you would like to associate this import with."), '#options' => $feed_types, '#default_value' => '', '#size' => 1, '#required' => TRUE, ); $node = new stdClass(); $node->uid = $user->uid; $form['#node'] = $node; $form['#method'] = 'post'; $form['#attributes']['enctype'] = 'multipart/form-data'; $form['submit'] = array('#type' => 'submit', '#value' => 'Submit'); if (module_exists('og')) { og_form_add_og_audience('feedapi_import_feeds_form', $form); } return $form; } /** * Handle the submission of the OPML import form */ function feedapi_import_feeds_form_submit($form_id, $form_values) { $file = file_check_upload('opml'); if ($file = file($file->filepath)) { $file = implode('', $file); if ($count = _feedapi_import_process($file, $form_values)) { drupal_set_message(t('Successfuly imported %count feeds from OPML', array('%count' => $count))); } else { drupal_set_message(t('Feed list could not be imported. Please check that this is an OPML file.'), 'error'); } } else { drupal_set_message(t('Data could not be retrieved, invalid or empty file.'), 'error'); } return 'admin/content/feed'; } /** * Delete expired items and return informations about the feed refreshing * * @param $feed * The feed object * @param $force * If TRUE, do not examine that the feed needs refresh * @return * FALSE if the feed don't have to be refreshed. (forbidden if the $force is TRUE) */ function feedapi_expire($feed, $force) { $needs_refresh = time() - $feed->checked > $feed->refresh; if (is_array($feed->items) && ($needs_refresh || $force) && $feed->items_delete != FEEDAPI_NEVER_DELETE_OLD) { foreach ($feed->items as $item) { if (isset($item->arrived) || isset($item->timestamp)) { $diff = abs(time() - (isset($item->arrived) ? $item->arrived : $item->timestamp)); if ($diff > $feed->items_delete) { foreach($feed->processors as $weight) { foreach ($weight as $processor) { module_invoke($processor, 'feedapi_item', 'delete', $item, $feed->nid); } } } } } } if ($needs_refresh == FALSE && $force == FALSE) { return FALSE; } return TRUE; } /** * Implementation of hook_form_alter(). */ function feedapi_form_alter($form_id, &$form) { // Content type form. if ($form_id == 'node_type_form' && isset($form['identity']['type'])) { if (isset($form['#post']['feedapi'])) { // TODO: Drupal automatically stores mutilated 'feedapi_'. $form['#node_type']->type - remove. $type = !empty($form['#node_type']->type) ? $form['#node_type']->type : $form['#post']['type']; _feedapi_store_settings(array('node_type' => $type), $form['#post']['feedapi']); } if (!$settings = _feedapi_get_settings(array('node_type' => $form['#node_type']->type))) { $settings = array(); } $form['feedapi'] = array( '#type' => 'fieldset', '#title' => t('Feed API'), '#collapsible' => TRUE, '#collapsed' => !$settings['enabled'], '#tree' => TRUE, ); $form['feedapi']['enabled'] = array( '#type' => 'checkbox', '#title' => t('Is a feed content type'), '#description' => t('Check this box if you want to use this content type for downloading feeds to your site.'), '#default_value' => $settings['enabled'], '#weight' => -15, ); $form['feedapi']['parsers'] = array( '#type' => 'fieldset', '#title' => t('Parser settings'), '#description' => t('Parsers turn a feed into an object ready for processing. Choose at least one.'), '#collapsible' => FALSE, '#tree' => TRUE, ); $parsers = module_implements('feedapi_feed', TRUE); rsort($parsers); foreach ($parsers as $parser) { $form['feedapi']['parsers'][$parser] = array( '#type' => 'fieldset', '#title' => feedapi_get_natural_name($parser), '#collapsible' => TRUE, '#collapsed' => !($settings['parsers'][$parser]['enabled'] || (count($parsers) == 1)), '#tree' => TRUE, '#weight' => $settings['parsers'][$parser]['weight'], ); $form['feedapi']['parsers'][$parser]['enabled'] = array( '#type' => 'checkbox', '#title' => t('Enable'), '#description' => t('Check this box if you want to enable the @name parser on this feed.', array('@name' => t($parser))), '#default_value' => $settings['parsers'][$parser]['enabled'] || (count($parsers) == 1), '#weight' => -15, ); $form['feedapi']['parsers'][$parser]['weight'] = array( '#type' => 'weight', '#delta' => 15, '#title' => t('Weight'), '#description' => t('Control the execution order. Parsers with lower weights are called before parsers with higher weights.'), '#default_value' => $settings['parsers'][$parser]['weight'], '#weight' => -14, ); } $form['feedapi']['processors'] = array( '#type' => 'fieldset', '#title' => t('Processor settings'), '#description' => t('Processors are any kind of add on modules that hook into the feed handling process on download time - you can decide here what should happen to feed items once they are downloaded and parsed.'), '#collapsible' => FALSE, '#tree' => TRUE, ); $processors = module_implements('feedapi_item', TRUE); rsort($processors); foreach ($processors as $processor) { $form['feedapi']['processors'][$processor] = array( '#type' => 'fieldset', '#title' => feedapi_get_natural_name($processor), '#collapsible' => TRUE, '#collapsed' => !($settings['processors'][$processor]['enabled'] || (count($processors) == 1)), '#tree' => TRUE, '#weight' => $settings['processors'][$processor]['weight'], ); $form['feedapi']['processors'][$processor]['enabled'] = array( '#type' => 'checkbox', '#title' => t('Enable'), '#description' => t('Check this box if you want to enable the @name processor on this feed.', array('@name' => t($processor))), '#default_value' => $settings['processors'][$processor]['enabled'], '#weight' => -15, ); $form['feedapi']['processors'][$processor]['weight'] = array( '#type' => 'weight', '#delta' => 15, '#title' => t('Weight'), '#description' => t('Control the execution order. Processors with lower weights are called before processors with higher weights.'), '#default_value' => $settings['processors'][$processor]['weight'], '#weight' => -14, ); } $form['feedapi'] = array_merge_recursive($form['feedapi'], _feedapi_invoke_settings_form('node_type_form', $form['#node_type']->type)); // We add again to ensure the correct order. Without it, the submit and delete buttons are in a wrong place (not last) $submit = $form['submit']; $delete = $form['delete']; unset($form['submit'], $form['delete']); $form['submit'] = $submit; $form['delete'] = $delete; } // FeedAPI-enabled node form. if ($form['type']['#value'] .'_node_form' == $form_id && _feedapi_is_enabled($form['type']['#value'])) { $form['title']['#required'] = FALSE; $form['title']['#description'] = t('This field will be populated with the feed title. You can override by filling in this field.'); $form['body_filter']['body']['#description'] = t('This field will be populated with the feed description. You can override by filling in this field.'); $form['feedapi'] = array( '#type' => 'fieldset', '#title' => t('Feed'), '#collapsible' => TRUE, '#collapsed' => FALSE, '#tree' => TRUE, ); $feed_url = $form['#node']->feed->url ? $form['#node']->feed->url : $form['#post']['feedapi']['feedapi_url']; $form['feedapi']['feedapi_url'] = array( '#type' => 'textfield', '#title' => t('Feed URL'), '#description' => t('Enter feed URL.'), '#default_value' => $feed_url, '#maxlength' => 255, ); // Prepopulate title and description if not set yet and URL available. if ($feed_url && !($form['#post']['title'] && $form['#post']['body'])) { // Todo: duplicate code - see nodeapi('validate') and 'submit' $feed = new stdClass(); $feed->url = $feed_url; $settings = _feedapi_get_settings(array('node_type' => $form['type']['#value'])); $assigned = _feedapi_assign_parser_processor($feed_url, $settings); $feed->processors = $assigned['processors']; $feed->parsers = $assigned['parsers']; $feed->feed_type = $assigned['type']; if (!empty($feed->feed_type)) { $feed = _feedapi_call_parsers($feed, $feed->parsers, FALSE); } if (!$form['#post']['title']) { $form['title']['#value'] = $feed->title; } if (!$form['#post']['body']) { $form['body_filter']['body']['#value'] = $feed->description; } } $period = drupal_map_assoc(array(3600, 10800, 21600, 32400, 43200, 86400, 172800, 259200, 604800, 1209600, 2419200, 3628800, 4838400, 7257600, 15724800, 31536000), 'format_interval'); $period[FEEDAPI_NEVER_DELETE_OLD] = t('Never'); $form['feedapi']['feedapi_items_delete'] = array( '#type' => 'select', '#title' => t('Delete news items older than'), '#options' => $period, '#default_value' => isset($form['#node']->feed->items_delete) ? $form['#node']->feed->items_delete : $form['#post']['feedapi_items_delete'], ); if (isset($form['#node']->feed->update_existing)) { $update_existing = $form['#node']->feed->update_existing; } elseif (isset($form['#post']['feedapi_update_existing'])) { $update_existing = $form['#post']['feedapi_update_existing']; } else { $update_existing = TRUE; } $update_existing = isset($form['#node']->feed->update_existing) ? $form['#node']->feed->update_existing : $form['#post']['feedapi_update_existing']; $form['feedapi']['feedapi_update_existing'] = array( '#type' => 'checkbox', '#title' => t('Update already existing news items'), '#default_value' => $update_existing, ); // Retrieve form snippets from add on modules. $form_snippets = _feedapi_invoke_settings_form('node_form', $form['type']['#value'], $form['#node']->nid); // Format snippets - make fieldsets out of them. foreach ($form_snippets as $k => $stage) { $form_snippets[$k]['#type'] = 'fieldset'; $form_snippets[$k]['#title'] = feedapi_get_natural_name($k); $form_snippets[$k]['#collapsible'] = TRUE; $form_snippets[$k]['#collapsed'] = TRUE; $form_snippets[$k]['#tree'] = TRUE; foreach (array_keys($stage) as $m) { $form_snippets[$k][$m]['#type'] = 'fieldset'; $form_snippets[$k][$m]['#title'] = feedapi_get_natural_name($m); $form_snippets[$k][$m]['#collapsible'] = TRUE; $form_snippets[$k][$m]['#collapsed'] = FALSE; $form_snippets[$k][$m]['#tree'] = TRUE; } } $form_snippets['#tree'] = TRUE; $form['feedapi'] = array_merge_recursive($form['feedapi'], $form_snippets); } } /** * Implementation of hook_cron(). */ function feedapi_cron() { $result = db_query("SELECT nid FROM {feedapi} ORDER BY checked ASC"); timer_start('feedapi_cron'); while ($feed = db_fetch_object($result, $row++)) { feedapi_invoke_feedapi('load', $feed); $timeout = feedapi_invoke_feedapi('refresh', $feed, TRUE); if ($timeout === FALSE) { break; } } } /** * Provide a UI for overviewing the existing feeds */ function feedapi_management_page() { $header = array( t('Title'), t('Last refresh'), t('New elements added per update'), t('Update rate'), t('Number of items in the feed'), t('Processing time'), t('Commands'), ); $rows = array(); $result = db_query("SELECT nid from {feedapi} ORDER BY checked DESC"); while ($nid = db_result($result, $row++)) { $node = node_load($nid); $commands = array(l(t('Delete'), 'node/'. $node->nid . '/delete', NULL, 'destination=admin/content/feed'), l(t('Remove items'), 'node/'. $node->nid . '/purge', NULL, 'destination=admin/content/feed'), l(t('Refresh'), 'node/'. $node->nid . '/refresh'), l(t('Edit'), 'node/'. $node->nid . '/edit'), ); $ext_commands = module_invoke_all('feedapi_edit_option', $node->feed); if (count($ext_commands) > 0) { foreach ($ext_commands as $command) { $commands[] = l($command['name'], $command['link']); } } if (count($node->feed->statistics['download_num']) != 0 && count($node->feed->statistics['new']) != 0 && count($node->feed->statistics['process_time']) != 0) { $update_rate = _feedapi_update_rate($node->feed->statistics['update_times']); $rows[] = array( l($node->title, "node/$node->nid"), $node->feed->checked == 0 ? t('Never') : format_interval(time() - $node->feed->checked) ." ". t("ago"), round(array_sum($node->feed->statistics['new']) / count($node->feed->statistics['new']), 2), is_numeric($update_rate) ? format_interval($update_rate) : $update_rate, round((array_sum($node->feed->statistics['download_num']) / count($node->feed->statistics['download_num'])), 2), round((array_sum($node->feed->statistics['process_time']) / count($node->feed->statistics['process_time'])), 2). ' '. t('ms'), theme('item_list', $commands), ); } else { $rows[] = array( l($node->title, "node/$node->nid"), $node->feed->checked == 0 ? t('Never') : format_interval(time() - $node->feed->checked) ." ". t("ago"), '', '', t('No enough data for statistics'), '', theme('item_list', $commands), ); } } $info = format_plural(variable_get('feedapi_statistics_queue', 10), "Average over the last refresh.", "Averages over the last @count refreshes." ); return $info. theme('table', $header, $rows). t(''). l(t('Export site feeds as OPML'), 'admin/content/feed/export_opml'); } /** * This is shown instead of normal node form when the simplified form is chosen at the settings */ function feedapi_simplified_form($type) { $form['feedapi']['type'] = array( '#type' => 'hidden', '#value' => $type ); $form['feedapi']['feedapi_url'] = array( '#title' => t('Feed URL'), '#type' => 'textfield', '#size' => 40, '#required' => TRUE, '#maxlength' => 255, ); $form['feedapi']['submit'] = array( '#type' => 'submit', '#value' => t('Submit'), ); $form['feedapi']['next'] = array( '#type' => 'submit', '#value' => t('Submit and edit'), ); return $form; } /** * Validate by feedapi_nodeapi */ function feedapi_simplified_form_validate($form_id, $form_values) { $node = (object) $form_values; feedapi_nodeapi($node, 'validate', NULL, NULL); } /** * Create the node object and save */ function feedapi_simplified_form_submit($form_id, $form_values) { $node = (object) $form_values; $node_options = variable_get('node_options_'. $node->type, array('status', 'promote')); // If this is a new node, fill in the default values. foreach (array('status', 'promote', 'sticky') as $key) { $node->$key = in_array($key, $node_options); } node_object_prepare($node); $node = node_submit($node); global $user; $node->uid = $user->uid; node_save($node); drupal_goto('node/'. $node->nid. ($form_values['op'] == t('Submit and edit') ? '/edit' : '')); } /** * Get the module-defined natural name of FeedAPI parser or processor * Define this name in hook_help(): * * function hook_help($section) { * switch ($section) { * case 'feedapi/full_name': * return t('Natural name'); * break; * } * } */ function feedapi_get_natural_name($module) { $help = $module .'_help'; $module_natural = function_exists($help) ? $help('feedapi/full_name') : t($module); return empty($module_natural) ? t($module) : $module_natural; } /** * Settings: * Allowed HTML tags, number of feeds refreshed in one round */ function feedapi_admin_settings() { $form['feedapi_allowed_html_tags'] = array( '#type' => 'textfield', '#title' => t('Allowed HTML tags'), '#size' => 80, '#maxlength' => 255, '#default_value' => variable_get('feedapi_allowed_html_tags', '

'), '#description' => t('The list of tags which are allowed in feeds, i.e., which will not be removed by Drupal.') ); $form['feedapi_allow_html_all'] = array( '#type' => 'checkbox', '#title' => t('Allow all HTML tags'), '#default_value' => variable_get('feedapi_allow_html_all', FALSE), '#description' => t('In this case the module does\'t filter any HTML elements from the incoming fields. This checkbox overrides the above list of allowed tags.') ); $form['feedapi_show_feed_origin_link'] = array( '#type' => 'checkbox', '#title' => t('Show originating site link with feed nodes.'), '#default_value' => variable_get('feedapi_show_feed_origin_link', TRUE), ); if (variable_get('feedapi_allow_html_all', FALSE)) { $form['feedapi_allowed_html_tags']['#disabled'] = TRUE; } $form['feedapi_statistics_queue'] = array( '#type' => 'select', '#title' => t('Number of samples stored per feed for statistics'), '#options' => drupal_map_assoc(array(10, 20, 50, 100)), '#default_value' => variable_get('feedapi_statistics_queue', 10), ); // Drupal will try to overwrite this value at cron time $max_exec = !ini_get('safe_mode') ? 240 : ini_get('max_execution_time'); $form['feedapi_cron_percentage'] = array( '#type' => 'select', '#title' => t('Cron time for FeedAPI [%]'), '#options' => drupal_map_assoc(array(15, 25, 50, 75)), '#default_value' => variable_get('feedapi_cron_percentage', 15), '#description' => t('Percentage of maximal PHP execution time (currently !exec seconds). At current settings, the FeedAPI cron process can run for up to !now seconds.', array("!exec" => $max_exec, "!now" => (variable_get('feedapi_cron_percentage', 15) / 100) * $max_exec)), ); return system_settings_form($form); } /** * Execute the enabled parsers and create an unified output * * @param $feed * Feed object * @param $cache * Parser result is cacheable or not * @param $parsers * Structure: array( * "primary" => "parser_primary", * "secondary" => array("parser1", "parser2", "parserN") * ); * @return * The object of the parser data */ function _feedapi_call_parsers($feed, $parsers, $cache) { $nid = $feed->nid; if (module_exists($parsers["primary"])) { $parser_output = module_invoke($parsers["primary"], 'feedapi_feed', 'parse', $feed, $cache); if ($parser_output === FALSE && $cache) { return $parser_output; } $feed = (object) array_merge((array) $feed, (array) $parser_output); } // Call the turned on parsers, create an union of returned options $parsers["secondary"] = is_array($parsers["secondary"]) ? $parsers["secondary"] : array(); foreach ($parsers["secondary"] as $parser) { $feed_ext = module_invoke($parser, 'feedapi_feed', 'parse', $feed, FALSE); $feed->options = (object) ((array) $feed->options + (array) $feed_ext->options); // Merge items' options foreach ($feed_ext->items as $key => $item) { $feed->items[$key]->options = (object) ((array) $feed->items[$key]->options + (array) $item->options); } } $feed->nid = $nid; foreach (module_implements('feedapi_after_parse') as $module) { $func = $module. '_feedapi_after_parse'; $func($feed); } // Filter bad or not allowed tags, sanitize data (currently timestamp checking) if (!variable_get('feedapi_allow_html_all', FALSE)) { $allowed = preg_split('/\s+|<|>/', variable_get('feedapi_allowed_html_tags', '


'), -1, PREG_SPLIT_NO_EMPTY); foreach (array('title', 'description') as $property) { if (isset($feed->{$property})) { if (is_string($feed->{$property})) { $feed->{$property} = filter_xss($feed->{$property}, $allowed); } } } for ($i = 0; $i < count($feed->items); $i++) { $feed->items[$i]->title = filter_xss($feed->items[$i]->title, $allowed); $feed->items[$i]->description = filter_xss($feed->items[$i]->description, $allowed); if ($feed->items[$i]->options->timestamp == 0) { $feed->items[$i]->options->timestamp = time(); } } } return $feed; } /** * Stores settings per content type or per node. */ function _feedapi_store_settings($args, $settings) { if ($args['nid']) { // Filter - only processors and parsers settings will be stored. $store_settings = array(); if ($settings['processors']) { $store_settings['processors'] = $settings['processors']; } if ($settings['parsers']) { $store_settings['parsers'] = $settings['parsers']; } db_query("UPDATE {feedapi} SET settings = '%s' WHERE nid = %d", serialize($store_settings), $args['nid']); module_invoke_all('feedapi_after_settings', $args['nid'], $settings); } elseif ($args['node_type']) { variable_set('feedapi_settings_'. $args['node_type'], $settings); } } /** * Returns TRUE if feedapi is enabled for given node type. */ function _feedapi_is_enabled($node_type) { $settings = _feedapi_get_settings(array('node_type' => $node_type)); return $settings['enabled'] ? TRUE : FALSE; } /** * Retrieve settings per content type or per node. * $args = array( * 'node_type' => '', * 'nid' => , * ) * Todo: add caching. Watch out: cache permutations of node_type or node_type+nid or nid seperately. */ function _feedapi_get_settings($args) { if (isset($args['nid'])) { if ($settings = db_fetch_object(db_query('SELECT settings FROM {feedapi} WHERE nid = %d', $args['nid']))) { $settings = unserialize($settings->settings); if (is_array($settings)) { return $settings; } } } // Fallback: node_type. if (isset($args['node_type'])) { if ($settings = variable_get('feedapi_settings_'. $args['node_type'], FALSE)) { return $settings; } } return FALSE; } /** * Set default value of $form elements if present in $settings. */ function _feedapi_populate($form, $settings) { static $i; $i++; foreach ($form as $k => $v) { if (is_array($v)) { if (array_key_exists('#default_value', $v)) { if (isset($settings[$k])) { $form[$k]['#default_value'] = $settings[$k]; $form[$k]['#disabled'] = user_access('advanced feedapi options') ? FALSE : TRUE; } } else if (!array_key_exists('#type', $v)){ $form[$k] = _feedapi_populate($form[$k], $settings[$k]); } } } $i--; return $form; } /** * Invokes feedapi_settings_form hook. * $args = array( * 'node_type' => '', * 'nid' => , * ) * Todo: simplify code, distinguish between content-type and node form * so that we can show different help text. */ function _feedapi_invoke_settings_form($form_type, $node_type = NULL, $nid = NULL) { $form = array(); switch ($form_type) { case 'node_type_form': if (!isset($node_type)) { drupal_set_message(t('Called _feedapi_invoke_settings_form without specifying $node_type'), 'error'); } $parsers = module_implements('feedapi_feed', TRUE); $content_type_settings = _feedapi_get_settings(array('node_type' => $node_type)); foreach($parsers as $module) { $result = array(); $result = module_invoke($module, 'feedapi_settings_form', 'parsers'); if (is_array($result)) { // Todo: clean up form_alter() content type form building and activate the line below // - we are basically building the form twice at the moment! // $result['#weight'] = $content_type_settings['parsers'][$module]['weight']; $form['parsers'][$module] = $result; } } $suitable_processors = module_implements('feedapi_item', TRUE); foreach($suitable_processors as $module) { $result = array(); $result = module_invoke($module, 'feedapi_settings_form', 'processors'); if (is_array($result)) { // Todo: clean up form_alter() content type form building and activate the line below. // $result['#weight'] = $content_type_settings['processors'][$module]['weight']; $form['processors'][$module] = $result; } } // Populate form with content type settings if available. if ($content_type_settings) { $form = _feedapi_populate($form, $content_type_settings); } return $form; case 'node_form': if (!isset($node_type)) { drupal_set_message(t('Called _feedapi_invoke_settings_form without specifying $node_type'), 'error'); } // Get settings for corresponding content type // Which parsers / processors are enabled is a per content-type setting. if ($content_type_settings = _feedapi_get_settings(array('node_type' => $node_type))) { // retrieve forms. $suitable_parsers = module_implements('feedapi_feed', TRUE); foreach ($suitable_parsers as $module) { if ($content_type_settings['parsers'][$module]['enabled']) { $result = array(); $result = module_invoke($module, 'feedapi_settings_form', 'parsers'); if (is_array($result)) { $result['#weight'] = $content_type_settings['parsers'][$module]['weight']; $form['parsers'][$module] = $result; } } } $suitable_processors = module_implements('feedapi_item', TRUE); foreach ($suitable_processors as $module) { if ($content_type_settings['processors'][$module]['enabled']) { $result = array(); $result = module_invoke($module, 'feedapi_settings_form', 'processors'); if (is_array($result)) { $result['#weight'] = $content_type_settings['processors'][$module]['weight']; $form['processors'][$module] = $result; } } } // If we are on a node form, get per node settings and populate form. if ($nid) { if ($settings = _feedapi_get_settings(array('nid' => $nid, 'node_type' => $node_type))) { $form = _feedapi_populate($form, $settings); } } // Else populate with presets. else { $form = _feedapi_populate($form, $content_type_settings); } } return $form; default: drupal_set_message(t('_feedapi_invoke_settings_form called with illegal arguments'), 'error'); break; } } /** * Auto-detect one possible parser and one feed and item processor to the given URL * It follows the order of the modules from the module's settings page * * @param $node */ function _feedapi_assign_parser_processor($url, $settings) { $parsers = module_implements('feedapi_feed', TRUE); // Get the parsers' weight $sorted_parsers = array(); foreach ($parsers as $parser) { if ($settings['parsers'][$parser]['enabled']) { $sorted_parsers[$settings['parsers'][$parser]['weight']][] = $parser; } } $success = FALSE; $keys = array_keys($sorted_parsers); if (count($keys) > 0) { $low = min($keys); $max = max($keys); // Call the ordered parsers to find all the compatible $parsers = array(); $parsers['secondary'] = array(); for ($i = $low; $i <= $max; $i++) { if (isset($sorted_parsers[$i])) { foreach ($sorted_parsers[$i] as $parser) { $type = module_invoke($parser, 'feedapi_feed', 'compatible', $url); if (!isset($parsers['primary']) && is_string($type)) { $success = TRUE; $parsers['primary'] = $parser; } else if (is_string($type)) { $parsers['secondary'][] = $parser; } } } } } if (!$success) { // We can't find a suitable parser at all return array('type' => '', 'parser' => '', 'processor_item' => ''); } // Get the feed item processor's weight $all_processors = module_implements('feedapi_item'); foreach ($all_processors as $processor) { if (in_array($type, module_invoke($processor, 'feedapi_item', 'type'))) { $processors[] = $processor; } } $processors = array_intersect($processors, array_keys($settings['processors'])); foreach ($processors as $processor) { if ($settings['processors'][$processor]['enabled']) { $sorted_processors[$settings['processors'][$processor]['weight']][] = $processor; } } return array( 'type' => $type, 'parsers' => $parsers, 'processors' => $sorted_processors ); } /** * Add the element of the queue. This queue is fixed-length, the first element is discarded if full. */ function _feedapi_queue($queue, $new_elem) { if (empty($queue) || !isset($queue)) { $queue = array(); } array_push($queue, $new_elem); if (count($queue) > variable_get('feedapi_statistics_queue', 10)) { array_shift($queue); } return $queue; } /** * Calculate the average between-update time */ function _feedapi_update_rate($update_times) { for ($i = 0; $i < count($update_times) - 1; $i++) { $between[] = abs($update_times[$i] - $update_times[$i + 1]); } return (count($between) > 0) ? round(array_sum($between) / count($between), 2) : t('No data yet'); } /** * Remove non-existing processors from the processors arrays */ function _feedapi_sanitize_processors(&$feed) { if (is_array($feed->processors) && count($feed->processors) > 0) { foreach ($feed->processors as $processors) { foreach ($processors as $key => $processor) { if (!module_exists($processor)) { unset($feed->processors[$key]); } } } } } /** * Generates an OPML representation of all feeds. */ function _feedapi_export_opml() { $result = db_query(db_rewrite_sql('SELECT n.title, f.url FROM {feedapi} f INNER JOIN {node} n ON f.nid = n.nid ORDER BY n.title ASC')); $output = "\n"; $output .= "\n"; $output .= "\n"; $output .= '\n"; $output .= ''. gmdate('r') ."\n"; $output .= "\n"; $output .= "\n"; while ($feed = db_fetch_object($result)) { $output .= ''."\n"; } $output .= "\n"; $output .= "\n"; drupal_set_header('Content-Type: text/xml; charset=utf-8'); print $output; exit(0); } /** * Imports from OPML XML file */ function _feedapi_import_process($opml, $args = array()) { $feeds = array(); $count = 0; $parser = drupal_xml_parser_create($opml); //Some OPML Files don't have the xml tag, which causes parsing to fail. Hence, using the appended version as a fallback parse if (xml_parse_into_struct($parser, $opml, $vals, $index) || xml_parse_into_struct($parser, ''.$opml, $vals, $index)) { foreach ($vals as $entry) { if ($entry['tag'] == 'OUTLINE') { $feeds[] = $entry['attributes']; } } foreach ($feeds as $feed) { if ($feed['XMLURL']) { // check if feed url is already in the list $dupe = db_num_rows(db_query("SELECT nid FROM {feedapi} WHERE url = '%s'", $feed['XMLURL'])); // If the feed is not already in the list, add it if (!$dupe) { // Generate a feed structure $node = new stdClass(); $node->title = $feed['TITLE']; $node->body = $feed['TEXT']; $node->type = $args['feed_type']; $node->feedapi_url = $feed['XMLURL']; $node->og_groups = $args['og_groups']; $node->og_public = (int) $args['og_public']; node_object_prepare($node); $node = node_submit($node); node_save($node); $count++; } } } } return $count; }