Unverified Commit fd8233c7 authored by alexpott's avatar alexpott

Issue #1927648 by damiankloip, Wim Leers, marthinal, tedbow, Arla, alexpott,...

Issue #1927648 by damiankloip, Wim Leers, marthinal, tedbow, Arla, alexpott, juampynr, garphy, bc, ibustos, eiriksm, larowlan, dawehner, gcardinal, vivekvpandya, kylebrowning, Sam152, neclimdul, pnagornyak, drnikki, gaurav.goyal, queenvictoria, kim.pepper, Berdir, clemens.tolboom, blainelang, moshe weitzman, linclark, webchick, Dave Reid, dabito, skyredwang, klausi, dagmar, gabesullice, pwolanin, amateescu, slashrsm, andypost, catch, aheimlich: Allow creation of file entities from binary data via REST requests
parent f7cd4d7d
......@@ -19,6 +19,11 @@
use Drupal\Core\Entity\EntityStorageInterface;
use Drupal\Core\Template\Attribute;
/**
* The regex pattern used when checking for insecure file types.
*/
define('FILE_INSECURE_EXTENSION_REGEX', '/\.(php|pl|py|cgi|asp|js)(\.|$)/i');
// Load all Field module hooks for File.
require_once __DIR__ . '/file.field.inc';
......@@ -954,7 +959,7 @@ function file_save_upload($form_field_name, $validators = [], $destination = FAL
// rename filename.php.foo and filename.php to filename.php.foo.txt and
// filename.php.txt, respectively). Don't rename if 'allow_insecure_uploads'
// evaluates to TRUE.
if (!\Drupal::config('system.file')->get('allow_insecure_uploads') && preg_match('/\.(php|pl|py|cgi|asp|js)(\.|$)/i', $file->getFilename()) && (substr($file->getFilename(), -4) != '.txt')) {
if (!\Drupal::config('system.file')->get('allow_insecure_uploads') && preg_match(FILE_INSECURE_EXTENSION_REGEX, $file->getFilename()) && (substr($file->getFilename(), -4) != '.txt')) {
$file->setMimeType('text/plain');
// The destination filename will also later be used to create the URI.
$file->setFilename($file->getFilename() . '.txt');
......
......@@ -127,8 +127,6 @@ protected function checkCreateAccess(AccountInterface $account, array $context,
// create file entities that are referenced from another entity
// (e.g. an image for a article). A contributed module is free to alter
// this to allow file entities to be created directly.
// @todo Update comment to mention REST module when
// https://www.drupal.org/node/1927648 is fixed.
return AccessResult::neutral();
}
......
<?php
namespace Drupal\file;
use Drupal\Core\DependencyInjection\ContainerBuilder;
use Drupal\Core\DependencyInjection\ServiceModifierInterface;
use Drupal\Core\StackMiddleware\NegotiationMiddleware;
/**
* Adds 'application/octet-stream' as a known (bin) format.
*/
class FileServiceProvider implements ServiceModifierInterface {
/**
* {@inheritdoc}
*/
public function alter(ContainerBuilder $container) {
if ($container->has('http_middleware.negotiation') && is_a($container->getDefinition('http_middleware.negotiation')->getClass(), NegotiationMiddleware::class, TRUE)) {
$container->getDefinition('http_middleware.negotiation')->addMethodCall('registerFormat', ['bin', ['application/octet-stream']]);
}
}
}
<?php
namespace Drupal\Tests\file\Functional;
use Drupal\Tests\rest\Functional\BasicAuthResourceTestTrait;
use Drupal\Tests\rest\Functional\FileUploadResourceTestBase;
/**
* @group file
*/
class FileUploadJsonBasicAuthTest extends FileUploadResourceTestBase {
use BasicAuthResourceTestTrait;
/**
* {@inheritdoc}
*/
public static $modules = ['basic_auth'];
/**
* {@inheritdoc}
*/
protected static $format = 'json';
/**
* {@inheritdoc}
*/
protected static $mimeType = 'application/json';
/**
* {@inheritdoc}
*/
protected static $auth = 'basic_auth';
}
<?php
namespace Drupal\Tests\file\Functional;
use Drupal\Tests\rest\Functional\CookieResourceTestTrait;
use Drupal\Tests\rest\Functional\FileUploadResourceTestBase;
/**
* @group file
*/
class FileUploadJsonCookieTest extends FileUploadResourceTestBase {
use CookieResourceTestTrait;
/**
* {@inheritdoc}
*/
protected static $format = 'json';
/**
* {@inheritdoc}
*/
protected static $mimeType = 'application/json';
/**
* {@inheritdoc}
*/
protected static $auth = 'cookie';
}
......@@ -15,7 +15,7 @@ services:
serializer.normalizer.file_entity.hal:
class: Drupal\hal\Normalizer\FileEntityNormalizer
deprecated: 'The "%service_id%" normalizer service is deprecated: it is obsolete, it only remains available for backwards compatibility.'
arguments: ['@entity.manager', '@http_client', '@hal.link_manager', '@module_handler', '@config.factory']
arguments: ['@entity.manager', '@hal.link_manager', '@module_handler', '@config.factory']
tags:
- { name: normalizer, priority: 20 }
serializer.normalizer.timestamp_item.hal:
......
......@@ -6,7 +6,6 @@
use Drupal\Core\Entity\EntityManagerInterface;
use Drupal\Core\Extension\ModuleHandlerInterface;
use Drupal\hal\LinkManager\LinkManagerInterface;
use GuzzleHttp\ClientInterface;
/**
* Converts the Drupal entity object structure to a HAL array structure.
......@@ -41,8 +40,6 @@ class FileEntityNormalizer extends ContentEntityNormalizer {
*
* @param \Drupal\Core\Entity\EntityManagerInterface $entity_manager
* The entity manager.
* @param \GuzzleHttp\ClientInterface $http_client
* The HTTP Client.
* @param \Drupal\hal\LinkManager\LinkManagerInterface $link_manager
* The hypermedia link manager.
* @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
......@@ -50,10 +47,9 @@ class FileEntityNormalizer extends ContentEntityNormalizer {
* @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
* The config factory.
*/
public function __construct(EntityManagerInterface $entity_manager, ClientInterface $http_client, LinkManagerInterface $link_manager, ModuleHandlerInterface $module_handler, ConfigFactoryInterface $config_factory) {
public function __construct(EntityManagerInterface $entity_manager, LinkManagerInterface $link_manager, ModuleHandlerInterface $module_handler, ConfigFactoryInterface $config_factory) {
parent::__construct($link_manager, $entity_manager, $module_handler);
$this->httpClient = $http_client;
$this->halSettings = $config_factory->get('hal.settings');
}
......@@ -73,16 +69,4 @@ public function normalize($entity, $format = NULL, array $context = []) {
return $data;
}
/**
* {@inheritdoc}
*/
public function denormalize($data, $class, $format = NULL, array $context = []) {
$file_data = (string) $this->httpClient->get($data['uri'][0]['value'])->getBody();
$path = 'temporary://' . drupal_basename($data['uri'][0]['value']);
$data['uri'] = file_unmanaged_save_data($file_data, $path);
return $this->entityManager->getStorage('file')->create($data);
}
}
......@@ -136,12 +136,4 @@ public function testGetBcUriField() {
$this->assertSame($this->baseUrl . '/' . $this->siteDirectory . '/files/drupal.txt', $actual['uri'][0]['value']);
}
/**
* {@inheritdoc}
*/
public function testPatch() {
// @todo https://www.drupal.org/node/1927648
$this->markTestSkipped();
}
}
<?php
namespace Drupal\Tests\hal\Functional\EntityResource\File;
use Drupal\Tests\rest\Functional\BasicAuthResourceTestTrait;
/**
* @group hal
*/
class FileUploadHalJsonBasicAuthTest extends FileUploadHalJsonTestBase {
use BasicAuthResourceTestTrait;
/**
* {@inheritdoc}
*/
public static $modules = ['basic_auth'];
/**
* {@inheritdoc}
*/
protected static $auth = 'basic_auth';
}
<?php
namespace Drupal\Tests\hal\Functional\EntityResource\File;
use Drupal\Tests\rest\Functional\CookieResourceTestTrait;
/**
* @group hal
*/
class FileUploadHalJsonCookieTest extends FileUploadHalJsonTestBase {
use CookieResourceTestTrait;
/**
* {@inheritdoc}
*/
protected static $auth = 'cookie';
}
<?php
namespace Drupal\Tests\hal\Functional\EntityResource\File;
use Drupal\Tests\rest\Functional\FileUploadResourceTestBase;
use Drupal\Tests\hal\Functional\EntityResource\HalEntityNormalizationTrait;
/**
* Tests binary data file upload route for HAL JSON.
*/
abstract class FileUploadHalJsonTestBase extends FileUploadResourceTestBase {
use HalEntityNormalizationTrait;
/**
* {@inheritdoc}
*/
public static $modules = ['hal'];
/**
* {@inheritdoc}
*/
protected static $format = 'hal_json';
/**
* {@inheritdoc}
*/
protected static $mimeType = 'application/hal+json';
/**
* {@inheritdoc}
*/
protected function getExpectedNormalizedEntity($fid = 1, $expected_filename = 'example.txt', $expected_as_filename = FALSE) {
$normalization = parent::getExpectedNormalizedEntity($fid, $expected_filename, $expected_as_filename);
// Cannot use applyHalFieldNormalization() as it uses the $entity property
// from the test class, which in the case of file upload tests, is the
// parent entity test entity for the file that's created.
// The HAL normalization adds entity reference fields to '_links' and
// '_embedded'.
unset($normalization['uid']);
return $normalization + [
'_links' => [
'self' => [
// @todo This can use a proper link once
// https://www.drupal.org/project/drupal/issues/2907402 is complete.
// This link matches what is generated from from File::url(), a
// resource URL is currently not available.
'href' => file_create_url($normalization['uri'][0]['value']),
],
'type' => [
'href' => $this->baseUrl . '/rest/type/file/file',
],
$this->baseUrl . '/rest/relation/file/file/uid' => [
['href' => $this->baseUrl . '/user/' . $this->account->id() . '?_format=hal_json']
],
],
'_embedded' => [
$this->baseUrl . '/rest/relation/file/file/uid' => [
[
'_links' => [
'self' => [
'href' => $this->baseUrl . '/user/' . $this->account->id() . '?_format=hal_json',
],
'type' => [
'href' => $this->baseUrl . '/rest/type/user/user',
],
],
'uuid' => [
[
'value' => $this->account->uuid(),
],
],
],
],
],
];
}
/**
* {@inheritdoc}
*
* @see \Drupal\Tests\hal\Functional\EntityResource\EntityTest\EntityTestHalJsonAnonTest::getNormalizedPostEntity()
*/
protected function getNormalizedPostEntity() {
return parent::getNormalizedPostEntity() + [
'_links' => [
'type' => [
'href' => $this->baseUrl . '/rest/type/entity_test/entity_test',
],
],
];
}
}
......@@ -151,6 +151,60 @@ protected function getExpectedNormalizedEntity() {
];
}
/**
* {@inheritdoc}
*/
protected function getExpectedNormalizedFileEntity() {
$normalization = parent::getExpectedNormalizedFileEntity();
$owner = static::$auth ? $this->account : User::load(0);
// Cannot use applyHalFieldNormalization() as it uses the $entity property
// from the test class, which in the case of file upload tests, is the
// parent entity test entity for the file that's created.
// The HAL normalization adds entity reference fields to '_links' and
// '_embedded'.
unset($normalization['uid']);
return $normalization + [
'_links' => [
'self' => [
// @todo This can use a proper link once
// https://www.drupal.org/project/drupal/issues/2907402 is complete.
// This link matches what is generated from from File::url(), a
// resource URL is currently not available.
'href' => file_create_url($normalization['uri'][0]['value']),
],
'type' => [
'href' => $this->baseUrl . '/rest/type/file/file',
],
$this->baseUrl . '/rest/relation/file/file/uid' => [
['href' => $this->baseUrl . '/user/' . $owner->id() . '?_format=hal_json']
],
],
'_embedded' => [
$this->baseUrl . '/rest/relation/file/file/uid' => [
[
'_links' => [
'self' => [
'href' => $this->baseUrl . '/user/' . $owner->id() . '?_format=hal_json',
],
'type' => [
'href' => $this->baseUrl . '/rest/type/user/user',
],
],
'uuid' => [
[
'value' => $owner->uuid(),
],
],
],
],
],
];
}
/**
* {@inheritdoc}
*/
......
<?php
namespace Drupal\Tests\hal\Functional;
use Drupal\file\Entity\File;
use Drupal\Tests\BrowserTestBase;
/**
* Tests that file entities can be denormalized in HAL.
*
* @group hal
* @see \Drupal\hal\Normalizer\FileEntityNormalizer
*/
class FileDenormalizeTest extends BrowserTestBase {
/**
* Modules to enable.
*
* @var array
*/
public static $modules = ['hal', 'file', 'node'];
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
// @todo Remove this work-around in https://www.drupal.org/node/1927648.
// @see hal_update_8501()
\Drupal::configFactory()
->getEditable('hal.settings')
->set('bc_file_uri_as_url_normalizer', TRUE)
->save(TRUE);
}
/**
* Tests file entity denormalization.
*/
public function testFileDenormalize() {
$file_params = [
'filename' => 'test_1.txt',
'uri' => 'public://test_1.txt',
'filemime' => 'text/plain',
'status' => FILE_STATUS_PERMANENT,
];
// Create a new file entity.
$file = File::create($file_params);
file_put_contents($file->getFileUri(), 'hello world');
$file->save();
$serializer = \Drupal::service('serializer');
$normalized_data = $serializer->normalize($file, 'hal_json');
$denormalized = $serializer->denormalize($normalized_data, 'Drupal\file\Entity\File', 'hal_json');
$this->assertTrue($denormalized instanceof File, 'A File instance was created.');
$this->assertIdentical('temporary://' . $file->getFilename(), $denormalized->getFileUri(), 'The expected file URI was found.');
$this->assertTrue(file_exists($denormalized->getFileUri()), 'The temporary file was found.');
$this->assertIdentical($file->uuid(), $denormalized->uuid(), 'The expected UUID was found');
$this->assertIdentical($file->getMimeType(), $denormalized->getMimeType(), 'The expected MIME type was found.');
$this->assertIdentical($file->getFilename(), $denormalized->getFilename(), 'The expected filename was found.');
$this->assertTrue($denormalized->isPermanent(), 'The file has a permanent status.');
// Try to denormalize with the file uri only.
$file_name = 'test_2.txt';
$file_path = 'public://' . $file_name;
file_put_contents($file_path, 'hello world');
$file_uri = file_create_url($file_path);
$data = [
'uri' => [
['value' => $file_uri],
],
];
$denormalized = $serializer->denormalize($data, 'Drupal\file\Entity\File', 'hal_json');
$this->assertIdentical('temporary://' . $file_name, $denormalized->getFileUri(), 'The expected file URI was found.');
$this->assertTrue(file_exists($denormalized->getFileUri()), 'The temporary file was found.');
$this->assertIdentical('text/plain', $denormalized->getMimeType(), 'The expected MIME type was found.');
$this->assertIdentical($file_name, $denormalized->getFilename(), 'The expected filename was found.');
$this->assertFalse($denormalized->isPermanent(), 'The file has a permanent status.');
}
}
......@@ -69,16 +69,51 @@ public static function create(ContainerInterface $container) {
* @param \Symfony\Component\HttpFoundation\Request $request
* The HTTP request object.
* @param \Drupal\rest\RestResourceConfigInterface $_rest_resource_config
* REST resource config entity ID.
* The REST resource config entity.
*
* @return \Drupal\rest\ResourceResponseInterface|\Symfony\Component\HttpFoundation\Response
* The REST resource response.
*/
public function handle(RouteMatchInterface $route_match, Request $request, RestResourceConfigInterface $_rest_resource_config) {
$response = $this->delegateToRestResourcePlugin($route_match, $request, $_rest_resource_config->getResourcePlugin());
$resource = $_rest_resource_config->getResourcePlugin();
$unserialized = $this->deserialize($route_match, $request, $resource);
$response = $this->delegateToRestResourcePlugin($route_match, $request, $unserialized, $resource);
return $this->prepareResponse($response, $_rest_resource_config);
}
/**
* Handles a REST API request without deserializing the request body.
*
* @param \Drupal\Core\Routing\RouteMatchInterface $route_match
* The route match.
* @param \Symfony\Component\HttpFoundation\Request $request
* The HTTP request object.
* @param \Drupal\rest\RestResourceConfigInterface $_rest_resource_config
* The REST resource config entity.
*
* @return \Symfony\Component\HttpFoundation\Response|\Drupal\rest\ResourceResponseInterface
* The REST resource response.
*/
public function handleRaw(RouteMatchInterface $route_match, Request $request, RestResourceConfigInterface $_rest_resource_config) {
$resource = $_rest_resource_config->getResourcePlugin();
$response = $this->delegateToRestResourcePlugin($route_match, $request, NULL, $resource);
return $this->prepareResponse($response, $_rest_resource_config);
}
/**
* Prepares the REST resource response.
*
* @param \Drupal\rest\ResourceResponseInterface $response
* The REST resource response.
* @param \Drupal\rest\RestResourceConfigInterface $resource_config
* The REST resource config entity.
*
* @return \Drupal\rest\ResourceResponseInterface
* The prepared REST resource response.
*/
protected function prepareResponse($response, RestResourceConfigInterface $resource_config) {
if ($response instanceof CacheableResponseInterface) {
$response->addCacheableDependency($_rest_resource_config);
$response->addCacheableDependency($resource_config);
// Add global rest settings config's cache tag, for BC flags.
// @see \Drupal\rest\Plugin\rest\resource\EntityResource::permissions()
// @see \Drupal\rest\EventSubscriber\RestConfigSubscriber
......@@ -181,14 +216,15 @@ protected function deserialize(RouteMatchInterface $route_match, Request $reques
* The route match.
* @param \Symfony\Component\HttpFoundation\Request $request
* The HTTP request object.
* @param mixed|null $unserialized
* The unserialized request body, if any.
* @param \Drupal\rest\Plugin\ResourceInterface $resource
* The REST resource plugin.
*
* @return \Symfony\Component\HttpFoundation\Response|\Drupal\rest\ResourceResponseInterface
* The REST resource response.
*/
protected function delegateToRestResourcePlugin(RouteMatchInterface $route_match, Request $request, ResourceInterface $resource) {
$unserialized = $this->deserialize($route_match, $request, $resource);
protected function delegateToRestResourcePlugin(RouteMatchInterface $route_match, Request $request, $unserialized, ResourceInterface $resource) {
$method = static::getNormalizedRequestMethod($route_match);
// Determine the request parameters that should be passed to the resource
......
......@@ -121,14 +121,14 @@ protected function getRoutesForResourceConfig(RestResourceConfigInterface $rest_
// The configuration has been validated, so we update the route to:
// - set the allowed response body content types/formats for methods
// that may send response bodies
// that may send response bodies (unless hardcoded by the plugin)
// - set the allowed request body content types/formats for methods that
// allow request bodies to be sent
// allow request bodies to be sent (unless hardcoded by the plugin)
// - set the allowed authentication providers
if (in_array($method, ['GET', 'HEAD', 'POST', 'PUT', 'PATCH'], TRUE)) {
if (in_array($method, ['GET', 'HEAD', 'POST', 'PUT', 'PATCH'], TRUE) && !$route->hasRequirement('_format')) {
$route->addRequirements(['_format' => implode('|', $rest_resource_config->getFormats($method))]);
}
if (in_array($method, ['POST', 'PATCH', 'PUT'], TRUE)) {
if (in_array($method, ['POST', 'PATCH', 'PUT'], TRUE) && !$route->hasRequirement('_content_type_format')) {
$route->addRequirements(['_content_type_format' => implode('|', $rest_resource_config->getFormats($method))]);
}
$route->setOption('_auth', $rest_resource_config->getAuthenticationProviders($method));
......
......@@ -537,13 +537,9 @@ public function testGet() {
// Note: deserialization of the XML format is not supported, so only test
// this for other formats.
if (static::$format !== 'xml') {
// @todo Work-around for HAL's FileEntityNormalizer::denormalize() being
// broken, being fixed in https://www.drupal.org/node/1927648, where this
// if-test should be removed.
if (!(static::$entityTypeId === 'file' && static::$format === 'hal_json')) {
$unserialized = $this->serializer->deserialize((string) $response->getBody(), get_class($this->entity), static::$format);
$this->assertSame($unserialized->uuid(), $this->entity->uuid());
}
$unserialized = $this->serializer->deserialize((string) $response->getBody(), get_class($this->entity), static::$format);
$this->assertSame($unserialized->uuid(), $this->entity->uuid());
}
// Finally, assert that the expected 'Link' headers are present.
if ($this->entity->getEntityType()->getLinkTemplates()) {
......@@ -745,27 +741,6 @@ protected static function castToString(array $normalization) {
return $normalization;
}
/**
* Recursively sorts an array by key.
*
* @param array $array
* An array to sort.
*
* @return array
* The sorted array.
*/
protected static function recursiveKSort(array &$array) {
// First, sort the main array.
ksort($array);
// Then check for child arrays.
foreach ($array as $key => &$value) {
if (is_array($value)) {
static::recursiveKSort($value);
}
}
}
/**
* Tests a POST request for an entity, plus edge cases to ensure good DX.
*/
......@@ -849,7 +824,13 @@ public function testPost() {
// DX: 403 when unauthorized.
$response = $this->request('POST', $url, $request_options);
$this->assertResourceErrorResponse(403, $this->getExpectedUnauthorizedAccessMessage('POST'), $response);
// @todo Remove this if-test in https://www.drupal.org/project/drupal/issues/2820364
if (static::$entityTypeId === 'media' && !static::$auth) {
$this->assertResourceErrorResponse(422, "Unprocessable Entity: validation failed.\nname: Name: this field cannot hold more than 1 values.\nfield_media_file.0: You do not have access to the referenced entity (file: 3).\n", $response);
}
else {
$this->assertResourceErrorResponse(403, $this->getExpectedUnauthorizedAccessMessage('POST'), $response);
}
$this->setUpAuthorization('POST');
......
......@@ -204,7 +204,12 @@ protected function getExpectedCacheContexts() {
* {@inheritdoc}
*/
public function testPost() {
// @todo https://www.drupal.org/node/1927648
// Drupal does not allow creating file entities independently. It allows you
// to create file entities that are referenced from another entity (e.g. an
// image for a node's image field).
// For that purpose, there is the "file_upload" REST resource plugin.
// @see \Drupal\file\FileAccessControlHandler::checkCreateAccess()
// @see \Drupal\file\Plugin\rest\resource\FileUploadResource
$this->markTestSkipped();
}
......
......@@ -2,12 +2,18 @@
namespace Drupal\Tests\rest\Functional\EntityResource\Media;
use Drupal\Component\Utility\NestedArray;
use Drupal\Core\Url;
use Drupal\file\Entity\File;
use Drupal\media\Entity\Media;
use Drupal\media\Entity\MediaType;
use Drupal\rest\RestResourceConfigInterface;
use Drupal\Tests\rest\Functional\BcTimestampNormalizerUnixTestTrait;
use Drupal\Tests\rest\Functional\EntityResource\EntityResourceTestBase;
use Drupal\user\Entity\Role;
use Drupal\user\Entity\User;
use Drupal\user\RoleInterface;
use GuzzleHttp\RequestOptions;
abstract class MediaResourceTestBase extends EntityResourceTestBase {
......@@ -45,7 +51,7 @@ protected function setUpAuthorization($method) {
break;
case 'POST':
$this->grantPermissionsToTestedRole(['create camelids media']);
$this->grantPermissionsToTestedRole(['create camelids media', 'access content']);
break;
case 'PATCH':
......@@ -230,9 +236,23 @@ protected function getNormalizedPostEntity() {
'value' => 'Dramallama',
],
],
'field_media_file' => [
[
'description' => NULL,
'display' => NULL,
'target_id' => 3,
],
],